/
percona_utils.py
1932 lines (1584 loc) · 65.5 KB
/
percona_utils.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
''' General utilities for percona '''
import collections
import datetime
import subprocess
from subprocess import Popen, PIPE
import socket
import tempfile
import copy
import os
import re
import shutil
import six
import uuid
from functools import partial
import time
import yaml
from charmhelpers.core.decorators import retry_on_exception
from charmhelpers.core.templating import render
from charmhelpers.core.host import (
lsb_release,
mkdir,
service,
pwgen,
CompareHostReleases,
)
from charmhelpers.core.hookenv import (
charm_dir,
unit_get,
relation_ids,
related_units,
relation_get,
relation_set,
local_unit,
service_name,
config,
log,
DEBUG,
INFO,
WARNING,
ERROR,
cached,
status_set,
network_get_primary_address,
application_version_set,
is_leader,
leader_get,
leader_set,
)
from charmhelpers.core.unitdata import kv
from charmhelpers.fetch import (
add_source,
apt_install,
apt_update,
filter_installed_packages,
get_upstream_version,
SourceConfigError,
)
from charmhelpers.contrib.network.ip import (
get_address_in_network,
get_ipv6_addr,
is_ip,
is_ipv6,
)
from charmhelpers.contrib.database.mysql import (
MySQLHelper,
)
from charmhelpers.contrib.hahelpers.cluster import (
is_clustered,
distributed_wait,
)
from charmhelpers.contrib.openstack.utils import (
make_assess_status_func,
pause_unit,
resume_unit,
is_unit_paused_set,
is_unit_upgrading_set,
)
# NOTE: python-mysqldb is installed by charmhelpers.contrib.database.mysql so
# hence why we import here
from MySQLdb import (
OperationalError
)
KEY = "keys/repo.percona.com"
REPO = """deb http://repo.percona.com/apt {release} main
deb-src http://repo.percona.com/apt {release} main"""
SEEDED_MARKER = "{data_dir}/seeded"
BACKUP_INFO = "{data_dir}/xtrabackup_info"
HOSTS_FILE = '/etc/hosts'
DEFAULT_MYSQL_PORT = 3306
INITIAL_CLUSTERED_KEY = 'initial-cluster-complete'
INITIAL_CLIENT_UPDATE_KEY = 'initial_client_update_done'
ADD_APT_REPOSITORY_FAILED = "add-apt-repository-failed"
# NOTE(ajkavanagh) - this is 'required' for the pause/resume code for
# maintenance mode, but is currently not populated as the
# charm_check_function() checks whether the unit is working properly.
REQUIRED_INTERFACES = {}
MYSQL_NAGIOS_CREDENTIAL_FILE = "/etc/nagios/mysql-check.cnf"
class LeaderNoBootstrapUUIDError(Exception):
"""Raised when the leader doesn't have set the bootstrap-uuid attribute"""
def __init__(self):
super(LeaderNoBootstrapUUIDError, self).__init__(
"the leader doesn't have set the bootstrap-uuid attribute")
class InconsistentUUIDError(Exception):
"""Raised when the leader and the unit have different UUIDs set"""
def __init__(self, leader_uuid, unit_uuid):
super(InconsistentUUIDError, self).__init__(
"Leader UUID ('{}') != Unit UUID ('{}')"
.format(leader_uuid, unit_uuid))
class DesyncedException(Exception):
'''Raised if PXC unit is not in sync with its peers'''
pass
class GRAStateFileNotFound(Exception):
"""Raised when the grastate file does not exist"""
pass
class FakeOSConfigRenderer(object):
"""This class is to provide to register_configs() as a 'fake'
OSConfigRenderer object that has a complete_contexts method that returns
an empty list. This is so that the pause/resume framework can be used
from charmhelpers that requires configs to be able to run.
This is a bit of a hack, but via Python's duck-typing enables the function
to work.
"""
def complete_contexts(self):
return []
def determine_packages():
if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'wily':
# NOTE(beisner): Use recommended mysql-client package
# https://launchpad.net/bugs/1476845
# https://launchpad.net/bugs/1571789
# NOTE(coreycb): This will install percona-xtradb-cluster-server-5.6
# for >= wily and percona-xtradb-cluster-server-5.7 for >= bionic.
return [
'percona-xtradb-cluster-server',
]
else:
return [
'percona-xtradb-cluster-server-5.5',
'percona-xtradb-cluster-client-5.5',
]
def seeded():
''' Check whether service unit is already seeded '''
return os.path.exists(SEEDED_MARKER.format(data_dir=resolve_data_dir()))
def mark_seeded():
''' Mark service unit as seeded '''
with open(SEEDED_MARKER.format(data_dir=resolve_data_dir()),
'w', encoding="utf-8") as seeded:
seeded.write('done')
def last_backup_sst():
""" Check if the last backup was an SST
The percona xtrabackup_info file (BACKUP_INFO) contains information about
the last backup/sync to this node, including the type of backup, either
incremental (IST) or full (SST).
We return True if we can successfully determine the last backup was SST
from the BACKUP_INFO file contents, otherwise we assume the last backup
was an incremental and return False.
@returns boolean
"""
result = False
try:
with open(BACKUP_INFO.format(data_dir=resolve_data_dir()), 'r') as f:
lines = f.readlines()
for line in lines:
if re.match('^incremental = N($|\n)', line):
result = True
except FileNotFoundError:
log("""Backup info file not found: %s, assuming last backup was
incremental""" %
BACKUP_INFO.format(data_dir=resolve_data_dir()), level=DEBUG)
except Exception:
log("""Unable to read backup info file: %s, assuming last backup was
incremental""" %
BACKUP_INFO.format(data_dir=resolve_data_dir()), level=DEBUG)
return result
def setup_percona_repo():
''' Configure service unit to use percona repositories '''
with open('/etc/apt/sources.list.d/percona.list', 'w') as sources:
sources.write(
REPO.format(
release=lsb_release()['DISTRIB_CODENAME']).encode('utf-8'))
subprocess.check_call(['apt-key', 'add', KEY])
def resolve_hostname_to_ip(hostname):
"""Resolve hostname to IP
@param hostname: hostname to be resolved
@returns IP address or None if resolution was not possible via DNS
"""
try:
import dns.resolver
except ImportError:
apt_install(filter_installed_packages(['python3-dnspython']),
fatal=True)
import dns.resolver
if config('prefer-ipv6'):
if is_ipv6(hostname):
return hostname
query_type = 'AAAA'
elif is_ip(hostname):
return hostname
else:
query_type = 'A'
# This may throw an NXDOMAIN exception; in which case
# things are badly broken so just let it kill the hook
answers = dns.resolver.query(hostname, query_type)
if answers:
return answers[0].address
def is_sufficient_peers():
"""Sufficient number of expected peers to build a complete cluster
If min-cluster-size has been provided, check that we have sufficient
number of peers as expected for a complete cluster.
If not defined assume a single unit.
@returns boolean
"""
min_size = config('min-cluster-size')
if min_size:
log("Checking for minimum of {} peer units".format(min_size),
level=DEBUG)
# Include this unit
units = 1
for rid in relation_ids('cluster'):
units += len(related_units(rid))
if units < min_size:
log("Insufficient number of peer units to form cluster "
"(expected={}, got={})".format(min_size, units), level=INFO)
return False
else:
log("Sufficient number of peer units to form cluster {}"
.format(min_size),
level=DEBUG)
return True
else:
log("min-cluster-size is not defined, race conditions may occur if "
"this is not a single unit deployment.", level=WARNING)
return True
def get_cluster_hosts():
"""Get the bootstrapped cluster peers
Determine the cluster peers that have bootstrapped and return the list
hosts. Secondarily, update the hosts file with IPv6 address name
resolution.
The returned host list is intended to be used in the
wsrep_cluster_address=gcomm:// setting. Therefore, the hosts must have
already been bootstrapped. If an un-bootstrapped host happens to be first
in the list, mysql will fail to start.
@side_effect update_hosts_file called for IPv6 hostname resolution
@returns list of hosts
"""
hosts_map = collections.OrderedDict()
local_cluster_address = get_cluster_host_ip()
# We need to add this localhost dns name to /etc/hosts along with peer
# hosts to ensure percona gets consistently resolved addresses.
if config('prefer-ipv6'):
addr = get_ipv6_addr(exc_list=[config('vip')], fatal=True)[0]
hosts_map = {addr: socket.gethostname()}
hosts = []
for relid in relation_ids('cluster'):
for unit in related_units(relid):
rdata = relation_get(unit=unit, rid=relid)
# NOTE(dosaboy): see LP: #1599447
cluster_address = rdata.get('cluster-address',
rdata.get('private-address'))
if config('prefer-ipv6'):
hostname = rdata.get('hostname')
if not hostname or hostname in hosts:
log("(unit=%s) Ignoring hostname '%s' provided by cluster "
"relation for addr %s" %
(unit, hostname, cluster_address), level=DEBUG)
continue
else:
log("(unit={}) hostname '{}' provided by cluster relation "
"for addr {}".format(unit, hostname, cluster_address),
level=DEBUG)
hosts_map[cluster_address] = hostname
host = hostname
else:
host = resolve_hostname_to_ip(cluster_address)
# Add only cluster peers who have set bootstrap-uuid
# An indiction they themselves are bootstrapped.
# Un-bootstrapped hosts in gcom lead mysql to fail to start
# if it happens to be the first address in the list
# Also fix strange bug when executed from actions where the local
# unit is returned in related_units. We do not want the local IP
# in the gcom hosts list.
if (rdata.get('bootstrap-uuid') and
host not in hosts and
host != local_cluster_address):
hosts.append(host)
if hosts_map:
update_hosts_file(hosts_map)
# Return a sorted list to avoid uneccessary restarts
return sorted(hosts)
SQL_SST_USER_SETUP = ("GRANT {permissions} ON *.* "
"TO 'sstuser'@'localhost' IDENTIFIED BY '{password}'")
SQL_SST_USER_SETUP_IPV6 = ("GRANT {permissions} "
"ON *.* TO 'sstuser'@'ip6-localhost' IDENTIFIED "
"BY '{password}'")
def get_db_helper():
return MySQLHelper(rpasswdf_template='/var/lib/charm/%s/mysql.passwd' %
(service_name()),
upasswdf_template='/var/lib/charm/%s/mysql-{}.passwd' %
(service_name()))
def configure_sstuser(sst_password):
# xtrabackup 2.4 (introduced in Bionic) needs PROCESS privilege for backups
permissions = [
"RELOAD",
"LOCK TABLES",
"REPLICATION CLIENT"
]
if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) >= 'bionic':
permissions.append('PROCESS')
m_helper = get_db_helper()
m_helper.connect(password=m_helper.get_mysql_root_password())
m_helper.execute(SQL_SST_USER_SETUP.format(
permissions=','.join(permissions),
password=sst_password)
)
m_helper.execute(SQL_SST_USER_SETUP_IPV6.format(
permissions=','.join(permissions),
password=sst_password)
)
# TODO: mysql charmhelper
def configure_mysql_root_password(password):
''' Configure debconf with root password '''
dconf = Popen(['debconf-set-selections'], stdin=PIPE)
# Set both percona and mysql password options to cover
# both upstream and distro packages.
packages = ["percona-server-server", "mysql-server",
"percona-xtradb-cluster-server"]
m_helper = get_db_helper()
root_pass = m_helper.get_mysql_root_password(password)
for package in packages:
dconf.stdin.write("{} {}/root_password password {}\n"
.format(package, package, root_pass)
.encode("utf-8"))
dconf.stdin.write("{} {}/root_password_again password {}\n"
.format(package, package, root_pass)
.encode("utf-8"))
dconf.communicate()
dconf.wait()
# TODO: Submit for charmhelper
def relation_clear(r_id=None):
''' Clears any relation data already set on relation r_id '''
settings = relation_get(rid=r_id,
unit=local_unit())
for setting in settings:
if setting not in ['public-address', 'private-address']:
settings[setting] = None
relation_set(relation_id=r_id,
**settings)
def update_hosts_file(_map):
"""Percona does not currently like ipv6 addresses so we need to use dns
names instead. In order to make them resolvable we ensure they are in
/etc/hosts.
See https://bugs.launchpad.net/galera/+bug/1130595 for some more info.
"""
with open(HOSTS_FILE, 'r', encoding="utf-8") as hosts:
lines = hosts.readlines()
log("Updating {} with: {} (current: {})".format(HOSTS_FILE, _map, lines),
level=DEBUG)
newlines = []
for ip, hostname in list(_map.items()):
if not ip or not hostname:
continue
keepers = []
for line in lines:
_line = line.split()
if len(line) < 2 or not (_line[0] == ip or hostname in _line[1:]):
keepers.append(line)
else:
log("Marking line '{}' for update or removal"
.format(line.strip()),
level=DEBUG)
lines = keepers
newlines.append("{} {}\n".format(ip, hostname))
lines += newlines
with tempfile.NamedTemporaryFile(delete=False) as tmpfile:
with open(tmpfile.name, 'w', encoding="utf-8") as hosts:
for line in lines:
hosts.write(line)
os.rename(tmpfile.name, HOSTS_FILE)
os.chmod(HOSTS_FILE, 0o644)
def assert_charm_supports_ipv6():
"""Check whether we are able to support charms ipv6."""
_release = lsb_release()['DISTRIB_CODENAME'].lower()
if CompareHostReleases(_release) < "trusty":
raise Exception("IPv6 is not supported in the charms for Ubuntu "
"versions less than Trusty 14.04")
def unit_sorted(units):
"""Return a sorted list of unit names."""
return sorted(
units, key=lambda a: int(a.split('/')[-1]))
def install_mysql_ocf():
dest_dir = '/usr/lib/ocf/resource.d/percona/'
for fname in ['ocf/percona/mysql_monitor']:
src_file = os.path.join(charm_dir(), fname)
if not os.path.isdir(dest_dir):
os.makedirs(dest_dir)
dest_file = os.path.join(dest_dir, os.path.basename(src_file))
if not os.path.exists(dest_file):
log('Installing %s' % dest_file, level='INFO')
shutil.copy(src_file, dest_file)
else:
log("'{}' already exists, skipping"
.format(dest_file), level='INFO')
# Always set to executable
os.chmod(dest_file, 0o755)
def get_wsrep_value(key):
m_helper = get_db_helper()
try:
m_helper.connect(password=m_helper.get_mysql_root_password())
except OperationalError:
log("Could not connect to db", DEBUG)
return None
cursor = m_helper.connection.cursor()
ret = None
try:
cursor.execute("show status like '{}'".format(key))
ret = cursor.fetchall()
except Exception as e:
log("Failed to get key={} '{}'".format(key, e), ERROR)
return None
finally:
cursor.close()
if ret:
return ret[0][1]
return None
def is_leader_bootstrapped():
""" Check that the leader is bootstrapped and has set required settings
:side_effect: calls leader_get
:returns: boolean
"""
check_settings = ['bootstrap-uuid', 'mysql.passwd', 'root-password',
'sst-password', 'leader-ip']
leader_settings = leader_get()
# Is the leader bootstrapped?
for setting in check_settings:
if leader_settings.get(setting) is None:
log("Leader is NOT bootstrapped {}: {}".format(setting,
leader_settings.get('bootstrap-uuid')), DEBUG)
return False
log("Leader is bootstrapped uuid: {}".format(
leader_settings.get('bootstrap-uuid')), DEBUG)
return True
def clustered_once():
"""Determine if the cluster has ever bootstrapped completely
Check unittest.kv if the cluster has bootstrapped at least once.
@returns boolean
"""
# Run is_bootstrapped once to guarantee kvstore is up to date
is_bootstrapped()
kvstore = kv()
return kvstore.get(INITIAL_CLUSTERED_KEY, False)
def is_bootstrapped():
"""Determine if each node in the cluster has been bootstrapped and the
cluster is complete with the expected number of peers.
Check that each node in the cluster, including this one, has set
bootstrap-uuid on the cluster relation.
Having min-cluster-size set will guarantee is_bootstrapped will not
return True until the expected number of peers are bootstrapped. If
min-cluster-size is not set, it will check peer relations to estimate the
expected cluster size. If min-cluster-size is not set and there are no
peers it must assume the cluster is bootstrapped in order to allow for
single unit deployments.
@returns boolean
"""
min_size = get_min_cluster_size()
if not is_sufficient_peers():
return False
elif min_size > 1:
uuids = []
for relation_id in relation_ids('cluster'):
units = related_units(relation_id) or []
units.append(local_unit())
for unit in units:
if not relation_get(attribute='bootstrap-uuid',
rid=relation_id,
unit=unit):
log("{} is not yet clustered".format(unit),
DEBUG)
return False
else:
bootstrap_uuid = relation_get(attribute='bootstrap-uuid',
rid=relation_id,
unit=unit)
if bootstrap_uuid:
uuids.append(bootstrap_uuid)
if len(uuids) < min_size:
log("Fewer than minimum cluster size: "
"{} percona units reporting clustered".format(min_size),
DEBUG)
return False
elif len(set(uuids)) > 1:
log("Found inconsistent bootstrap uuids: "
"{}".format(uuids), level=WARNING)
return False
else:
log("All {} percona units reporting clustered".format(min_size),
DEBUG)
elif not seeded():
# Single unit deployment but not yet bootstrapped
return False
# Set INITIAL_CLUSTERED_KEY as the cluster has fully bootstrapped
kvstore = kv()
if not kvstore.get(INITIAL_CLUSTERED_KEY, False):
kvstore.set(key=INITIAL_CLUSTERED_KEY, value=True)
kvstore.flush()
return True
def bootstrap_pxc():
"""Bootstrap PXC
On systemd systems systemctl bootstrap-pxc mysql does not work.
Run service mysql bootstrap-pxc to bootstrap."""
service('stop', 'mysql')
bootstrapped = service('bootstrap-pxc', 'mysql')
if not bootstrapped:
try:
cmp_os = CompareHostReleases(
lsb_release()['DISTRIB_CODENAME']
)
if cmp_os < 'bionic':
# NOTE(jamespage): execute under systemd-run to ensure
# that the bootstrap-pxc mysqld does
# not end up in the juju unit daemons
# cgroup scope.
cmd = ['systemd-run', '--service-type=forking',
'service', 'mysql', 'bootstrap-pxc']
subprocess.check_call(cmd)
else:
service('start', 'mysql@bootstrap')
except subprocess.CalledProcessError as e:
msg = 'Bootstrap PXC failed'
error_msg = '{}: {}'.format(msg, e)
status_set('blocked', msg)
log(error_msg, ERROR)
raise Exception(error_msg)
if CompareHostReleases(lsb_release()['DISTRIB_CODENAME']) < 'bionic':
# To make systemd aware mysql is running after a bootstrap
service('start', 'mysql')
log("Bootstrap PXC Succeeded", DEBUG)
def notify_bootstrapped(cluster_rid=None, cluster_uuid=None):
if cluster_rid:
rids = [cluster_rid]
else:
rids = relation_ids('cluster')
if not rids:
log("No relation ids found for 'cluster'", level=INFO)
return
if not cluster_uuid:
cluster_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
if not cluster_uuid:
cluster_uuid = str(uuid.uuid4())
log("Could not determine cluster uuid so using '%s' instead" %
(cluster_uuid), INFO)
log("Notifying peers that percona is bootstrapped (uuid=%s)" %
(cluster_uuid), DEBUG)
for rid in rids:
relation_set(relation_id=rid, **{'bootstrap-uuid': cluster_uuid})
if is_leader():
leader_set(**{'bootstrap-uuid': cluster_uuid})
def update_bootstrap_uuid():
"""This function verifies if the leader has set the bootstrap-uuid
attribute to then check it against the running cluster uuid, if the check
succeeds the bootstrap-uuid field is set in the cluster relation.
:returns: True if the cluster UUID was updated, False if the local UUID is
empty.
"""
lead_cluster_state_uuid = leader_get('bootstrap-uuid')
if not lead_cluster_state_uuid:
log('Leader has not set bootstrap-uuid', level=DEBUG)
raise LeaderNoBootstrapUUIDError()
wsrep_ready = get_wsrep_value('wsrep_ready') or ""
log("wsrep_ready: '{}'".format(wsrep_ready), DEBUG)
if wsrep_ready.lower() in ['on', 'ready']:
cluster_state_uuid = get_wsrep_value('wsrep_cluster_state_uuid')
else:
cluster_state_uuid = None
if not cluster_state_uuid:
log("UUID is empty: '{}'".format(cluster_state_uuid), level=DEBUG)
return False
elif lead_cluster_state_uuid != cluster_state_uuid:
# this may mean 2 things:
# 1) the units have diverged, which it's bad and we do stop.
# 2) cluster_state_uuid could not be retrieved because it
# hasn't been bootstrapped, mysqld is stopped, etc.
log('bootstrap uuid differs: %s != %s' % (lead_cluster_state_uuid,
cluster_state_uuid),
level=ERROR)
raise InconsistentUUIDError(lead_cluster_state_uuid,
cluster_state_uuid)
for rid in relation_ids('cluster'):
notify_bootstrapped(cluster_rid=rid,
cluster_uuid=cluster_state_uuid)
return True
def cluster_in_sync():
'''
Determines whether the current unit is in sync
with the rest of the cluster
'''
ready = get_wsrep_value('wsrep_ready') or False
sync_status = get_wsrep_value('wsrep_local_state') or 0
if ready and int(sync_status) in [2, 4]:
return True
return False
def charm_check_func(ensure_seeded=False):
"""Custom function to assess the status of the current unit
:param ensure_seeded: if set, then ensure that the seeded file exists.
This is to work around bug: #1868326 where the resume action to
manage_payload_services causes the seeded file to be removed under
certain circumstances. This is one of the only places where we can
actually re-assert it prior to the resume failing, as this function
gets a chance to run after management_payload_services() but prior to
the actual
:type ensure_seeded: bool
:returns: (status, message)
:rtype: Tuple[str, str]
"""
# Ensure seeded file is replaced if told or after any SST event post
# bootstrap. resolves bug #2000107
if (not seeded() and
(ensure_seeded or (is_bootstrapped() and last_backup_sst()))):
log("'seeded' file is missing but should exists; putting it back.")
mark_seeded()
if is_unit_upgrading_set():
# Avoid looping through attempting to determine cluster_in_sync
return ("blocked", "Unit upgrading.")
kvstore = kv()
# Using INITIAL_CLIENT_UPDATE_KEY as this is a step beyond merely
# clustered, but rather clustered and clients were previously notified.
if (kvstore.get(INITIAL_CLIENT_UPDATE_KEY, False) and
not check_mysql_connection()):
return ('blocked',
'MySQL is down. Sequence Number: {}. Safe To Bootstrap: {}'
.format(get_grastate_seqno(),
get_grastate_safe_to_bootstrap()))
@retry_on_exception(num_retries=10,
base_delay=2,
exc_type=DesyncedException)
def _cluster_in_sync():
'''Helper func to wait for a while for resync to occur
@raise DesynedException: raised if local unit is not in sync
with its peers
'''
if not cluster_in_sync():
raise DesyncedException()
min_size = config('min-cluster-size')
# Ensure that number of peers > cluster size configuration
if not is_sufficient_peers():
return ('blocked', 'Insufficient peers to bootstrap cluster')
if min_size and int(min_size) > 1:
# Once running, ensure that cluster is in sync
# and has the required peers
if not is_bootstrapped():
return ('waiting', 'Unit waiting for cluster bootstrap')
elif not seeded():
return ('waiting',
"Unit waiting to bootstrap ('seeded' file missing)")
elif cluster_ready():
try:
_cluster_in_sync()
return ('active', 'Unit is ready and clustered')
except DesyncedException:
return ('blocked', 'Unit is not in sync')
else:
return ('waiting', 'Unit waiting on hacluster relation')
else:
if seeded():
return ('active', 'Unit is ready')
else:
return ('waiting', 'Unit waiting to bootstrap')
@cached
def resolve_data_dir():
_release = lsb_release()['DISTRIB_CODENAME'].lower()
if CompareHostReleases(_release) < 'vivid':
return '/var/lib/mysql'
else:
return '/var/lib/percona-xtradb-cluster'
@cached
def resolve_cnf_file():
_release = lsb_release()['DISTRIB_CODENAME'].lower()
if CompareHostReleases(_release) < 'vivid':
return '/etc/mysql/my.cnf'
else:
return '/etc/mysql/percona-xtradb-cluster.conf.d/mysqld.cnf'
def register_configs():
"""Return a OSConfigRenderer object.
However, ceph-mon wasn't written using OSConfigRenderer objects to do the
config files, so this just returns an empty OSConfigRenderer object.
@returns empty FakeOSConfigRenderer object.
"""
return FakeOSConfigRenderer()
def services():
"""Return a list of services that are managed by this charm.
@returns [services] - list of strings that are service names.
"""
# NOTE(jamespage): Native systemd variants of the packagin
# use mysql@bootstrap to seed the cluster
# however this is cleared after a reboot,
# so dynamically check to see if this active
if service('is-active', 'mysql@bootstrap'):
return ['mysql@bootstrap']
return ['mysql']
def assess_status(configs):
"""Assess status of current unit
Decides what the state of the unit should be based on the current
configuration.
SIDE EFFECT: calls set_os_workload_status(...) which sets the workload
status of the unit.
Also calls status_set(...) directly if paused state isn't complete.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
kvstore = kv()
if kvstore.get(ADD_APT_REPOSITORY_FAILED, False):
# NOTE (rgildein): prevent unit status from changing from blocked
# to "Unit is ready", if adding a new source failed
log("skip assess_status, because adding new source failed", level=INFO)
status_set(
"blocked", "problem adding new source: {}".format(config("source"))
)
return
assess_status_func(configs)()
if pxc_installed():
# NOTE(fnordahl) ensure we do not call application_version_set with
# None argument. New charm deployments will have the meta-package
# installed, but upgraded deployments will not.
def _possible_packages():
base = determine_packages()[0]
yield base
if '.' not in base:
for i in range(5, 7 + 1):
yield base + '-5.' + str(i)
version = None
for pkg in _possible_packages():
version = get_upstream_version(pkg)
if version is not None:
break
else:
log('Unable to determine installed version for package "{}"'
.format(determine_packages()[0]), level=WARNING)
return
application_version_set(version)
def assess_status_func(configs):
"""Helper function to create the function that will assess_status() for
the unit.
Uses charmhelpers.contrib.openstack.utils.make_assess_status_func() to
create the appropriate status function and then returns it.
Used directly by assess_status() and also for pausing and resuming
the unit.
NOTE(ajkavanagh) ports are not checked due to race hazards with services
that don't behave sychronously w.r.t their service scripts. e.g.
apache2.
@param configs: a templating.OSConfigRenderer() object
@return f() -> None : a function that assesses the unit's workload status
"""
# BUG: 1868326 - the resume action on the mysql service can wipe out the
# seeded file; if it is seeded, then we put it back. This function is
# called as part of the resume_unit() call and is one of the only places to
# actually do and fix this check.
is_seeded = seeded()
return make_assess_status_func(
configs, REQUIRED_INTERFACES,
charm_func=lambda _: charm_check_func(ensure_seeded=is_seeded),
services=services(), ports=None)
def pause_unit_helper(configs):
"""Helper function to pause a unit, and then call assess_status(...) in
effect, so that the status is correctly updated.
Uses charmhelpers.contrib.openstack.utils.pause_unit() to do the work.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
_pause_resume_helper(pause_unit, configs)
def resume_unit_helper(configs):
"""Helper function to resume a unit, and then call assess_status(...) in
effect, so that the status is correctly updated.
Uses charmhelpers.contrib.openstack.utils.resume_unit() to do the work.
@param configs: a templating.OSConfigRenderer() object
@returns None - this function is executed for its side-effect
"""
_pause_resume_helper(resume_unit, configs)
def _pause_resume_helper(f, configs):
"""Helper function that uses the make_assess_status_func(...) from
charmhelpers.contrib.openstack.utils to create an assess_status(...)
function that can be used with the pause/resume of the unit
@param f: the function to be used with the assess_status(...) function
@returns None - this function is executed for its side-effect
"""
# TODO(ajkavanagh) - ports= has been left off because of the race hazard
# that exists due to service_start()
f(assess_status_func(configs),
services=services(),
ports=None)
def create_binlogs_directory():
if not pxc_installed():
log("PXC not yet installed. Not setting up binlogs", DEBUG)
return
binlogs_directory = os.path.dirname(config('binlogs-path'))
data_dir = resolve_data_dir() + '/'
if binlogs_directory.startswith(data_dir):
raise Exception("Configured binlogs directory ({}) must not be inside "
"mysql data dir".format(binlogs_directory))
if not os.path.isdir(binlogs_directory):
mkdir(binlogs_directory, 'mysql', 'mysql', 0o750)
def get_cluster_host_ip():
"""Get the this host's IP address for use with percona cluster peers
@returns IP to pass to cluster peers
"""
cluster_network = config('cluster-network')
if cluster_network:
cluster_addr = get_address_in_network(cluster_network, fatal=True)
else:
try:
cluster_addr = network_get_primary_address('cluster')
except NotImplementedError:
# NOTE(jamespage): fallback to previous behaviour
cluster_addr = resolve_hostname_to_ip(
unit_get('private-address')
)
return cluster_addr
def get_min_cluster_size():
""" Get the minimum cluster size
If the config value is set use that, if not count the number of units on
the cluster relation.
"""
min_cluster_size = config('min-cluster-size')
if not min_cluster_size:
units = 1
for relation_id in relation_ids('cluster'):
units += len(related_units(relation_id))
min_cluster_size = units
return min_cluster_size
def cluster_ready():
"""Determine if each node in the cluster is ready to respond to client
requests.
Once cluster_ready returns True it is safe to execute client relation
hooks.
If a VIP is set do not return ready until hacluster relationship is
complete.