Skip to content

Commit

Permalink
Install xtrabackup in percona docker image
Browse files Browse the repository at this point in the history 
Fix how we set passwords in integ tests to work for different versions
of mysql

Signed-off-by: deepthi <deepthi@planetscale.com>
  • Loading branch information
@deepthi
deepthi committed Jun 1, 2019
1 parent 5b25335 commit a549a40
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 56 deletions.
9 changes: 6 additions & 3 deletions docker/bootstrap/Dockerfile.percona
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,15 @@ RUN for i in $(seq 1 10); do apt-key adv --no-tty --keyserver keys.gnupg.net --r
} | debconf-set-selections && \
apt-get update && \
apt-get install -y --no-install-recommends \
percona-server-server-5.6 libperconaserverclient18.1-dev && \
rm -rf /var/lib/apt/lists/*
percona-server-server-5.6 libperconaserverclient18.1-dev rsync libev4 && \
rm -rf /var/lib/apt/lists/* && \
wget https://www.percona.com/downloads/XtraBackup/Percona-XtraBackup-2.4.13/binary/debian/stretch/x86_64/percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
dpkg -i percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb && \
rm -f percona-xtrabackup-24_2.4.13-1.stretch_amd64.deb

# Bootstrap Vitess
WORKDIR /vt/src/vitess.io/vitess

ENV MYSQL_FLAVOR MySQL56
USER vitess
RUN ./bootstrap.sh
RUN ./bootstrap.sh
42 changes: 25 additions & 17 deletions test/backup.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import environment
import tablet
import utils
from mysql_flavor import mysql_flavor

use_mysqlctld = False
use_xtrabackup = False
Expand Down Expand Up @@ -60,23 +61,6 @@ def setUpModule():
try:
environment.topo_server().setup()

# Create a new init_db.sql file that sets up passwords for all users.
# Then we use a db-credentials-file with the passwords.
new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
with open(environment.vttop + '/config/init_db.sql') as fd:
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write('''
# Set real passwords for all users.
ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
FLUSH PRIVILEGES;
''')
credentials = {
'vt_dba': ['VtDbaPass'],
'vt_app': ['VtAppPass'],
Expand All @@ -88,6 +72,30 @@ def setUpModule():
with open(db_credentials_file, 'w') as fd:
fd.write(json.dumps(credentials))

# Determine which column is used for user passwords in this MySQL version.
proc = tablet_master.init_mysql()
if use_mysqlctld:
tablet_master.wait_for_mysqlctl_socket()
else:
utils.wait_procs([proc])
try:
tablet_master.mquery('mysql', 'select password from mysql.user limit 0',
user='root')
password_col = 'password'
except MySQLdb.DatabaseError:
password_col = 'authentication_string'
utils.wait_procs([tablet_master.teardown_mysql()])
tablet_master.remove_tree(ignore_options=True)

# Create a new init_db.sql file that sets up passwords for all users.
# Then we use a db-credentials-file with the passwords.
new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
with open(environment.vttop + '/config/init_db.sql') as fd:
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write(mysql_flavor().change_passwords(password_col))

# start mysql instance external to the test
setup_procs = [
tablet_master.init_mysql(init_db=new_init_db,
Expand Down
33 changes: 14 additions & 19 deletions test/initial_sharding_multi.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,8 @@
import environment
import tablet
import utils

from mysql_flavor import mysql_flavor

# initial shard, covers everything
ks1_shard_master = tablet.Tablet(vt_dba_passwd='VtDbaPass')
ks1_shard_replica = tablet.Tablet(vt_dba_passwd='VtDbaPass')
Expand Down Expand Up @@ -100,6 +101,17 @@ def setUpModule():
global new_init_db, db_credentials_file

try:
credentials = {
'vt_dba': ['VtDbaPass'],
'vt_app': ['VtAppPass'],
'vt_allprivs': ['VtAllprivsPass'],
'vt_repl': ['VtReplPass'],
'vt_filtered': ['VtFilteredPass'],
}
db_credentials_file = environment.tmproot+'/db_credentials.json'
with open(db_credentials_file, 'w') as fd:
fd.write(json.dumps(credentials))

# Determine which column is used for user passwords in this MySQL version.
proc = ks1_shard_master.init_mysql()
utils.wait_procs([proc])
Expand All @@ -119,14 +131,8 @@ def setUpModule():
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write(mysql_flavor().change_passwords(password_col))
fd.write('''
# Set real passwords for all users.
ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
# connecting through a port requires 127.0.0.1
# --host=localhost will connect through socket
Expand Down Expand Up @@ -162,17 +168,6 @@ def setUpModule():
FLUSH PRIVILEGES;
''')
credentials = {
'vt_dba': ['VtDbaPass'],
'vt_app': ['VtAppPass'],
'vt_allprivs': ['VtAllprivsPass'],
'vt_repl': ['VtReplPass'],
'vt_filtered': ['VtFilteredPass'],
}
db_credentials_file = environment.tmproot+'/db_credentials.json'
with open(db_credentials_file, 'w') as fd:
fd.write(json.dumps(credentials))

setup_procs = [t.init_mysql(use_rbr=True, init_db=new_init_db,
extra_args=['-db-credentials-file',
db_credentials_file]) for t in all_mysql_tablets]
Expand Down
30 changes: 30 additions & 0 deletions test/mysql_flavor.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,24 @@ def disable_binlog_checksum(self, tablet):
"""Disables binlog_checksum if the flavor supports it."""
tablet.mquery("", "SET @@global.binlog_checksum=0")

def change_passwords(self, password_col):
"""set real passwords for all users"""
return '''
# Set real passwords for all users.
UPDATE mysql.user SET %s = PASSWORD('RootPass')
WHERE User = 'root' AND Host = 'localhost';
UPDATE mysql.user SET %s = PASSWORD('VtDbaPass')
WHERE User = 'vt_dba' AND Host = 'localhost';
UPDATE mysql.user SET %s = PASSWORD('VtAppPass')
WHERE User = 'vt_app' AND Host = 'localhost';
UPDATE mysql.user SET %s = PASSWORD('VtAllprivsPass')
WHERE User = 'vt_allprivs' AND Host = 'localhost';
UPDATE mysql.user SET %s = PASSWORD('VtReplPass')
WHERE User = 'vt_repl' AND Host = '%%';
UPDATE mysql.user SET %s = PASSWORD('VtFilteredPass')
WHERE User = 'vt_filtered' AND Host = 'localhost';
FLUSH PRIVILEGES;
''' % tuple([password_col] * 6)

class MariaDB(MysqlFlavor):
"""Overrides specific to MariaDB."""
Expand Down Expand Up @@ -169,6 +187,18 @@ class MySQL80(MySQL56):
"""Overrides specific to MySQL 8.0."""
def extra_my_cnf(self):
return environment.vttop + "/config/mycnf/master_mysql80.cnf"
def change_passwords(self, password_col):
"""set real passwords for all users"""
return '''
# Set real passwords for all users.
ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
FLUSH PRIVILEGES;
'''


# Map of registered MysqlFlavor classes (keyed by an identifier).
Expand Down
42 changes: 25 additions & 17 deletions test/xtrabackup_xtra.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import environment
import tablet
import utils
from mysql_flavor import mysql_flavor

use_mysqlctld = False
use_xtrabackup = True
Expand Down Expand Up @@ -60,23 +61,6 @@ def setUpModule():
try:
environment.topo_server().setup()

# Create a new init_db.sql file that sets up passwords for all users.
# Then we use a db-credentials-file with the passwords.
new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
with open(environment.vttop + '/config/init_db.sql') as fd:
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write('''
# Set real passwords for all users.
ALTER USER 'root'@'localhost' IDENTIFIED BY 'RootPass';
ALTER USER 'vt_dba'@'localhost' IDENTIFIED BY 'VtDbaPass';
ALTER USER 'vt_app'@'localhost' IDENTIFIED BY 'VtAppPass';
ALTER USER 'vt_allprivs'@'localhost' IDENTIFIED BY 'VtAllPrivsPass';
ALTER USER 'vt_repl'@'%' IDENTIFIED BY 'VtReplPass';
ALTER USER 'vt_filtered'@'localhost' IDENTIFIED BY 'VtFilteredPass';
FLUSH PRIVILEGES;
''')
credentials = {
'vt_dba': ['VtDbaPass'],
'vt_app': ['VtAppPass'],
Expand All @@ -88,6 +72,30 @@ def setUpModule():
with open(db_credentials_file, 'w') as fd:
fd.write(json.dumps(credentials))

# Determine which column is used for user passwords in this MySQL version.
proc = tablet_master.init_mysql()
if use_mysqlctld:
tablet_master.wait_for_mysqlctl_socket()
else:
utils.wait_procs([proc])
try:
tablet_master.mquery('mysql', 'select password from mysql.user limit 0',
user='root')
password_col = 'password'
except MySQLdb.DatabaseError:
password_col = 'authentication_string'
utils.wait_procs([tablet_master.teardown_mysql()])
tablet_master.remove_tree(ignore_options=True)

# Create a new init_db.sql file that sets up passwords for all users.
# Then we use a db-credentials-file with the passwords.
new_init_db = environment.tmproot + '/init_db_with_passwords.sql'
with open(environment.vttop + '/config/init_db.sql') as fd:
init_db = fd.read()
with open(new_init_db, 'w') as fd:
fd.write(init_db)
fd.write(mysql_flavor().change_passwords(password_col))

# start mysql instance external to the test
setup_procs = [
tablet_master.init_mysql(init_db=new_init_db,
Expand Down

0 comments on commit a549a40

Please sign in to comment.