Skip to content

Mysql 8 dev #602

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Mysql 8 dev #602

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fdac9a9
Fix MySQL 8.0 support
AMecea Sep 12, 2019
adfcaef
fix:#363(file ./sys_operator/status.ibd is missing)
zhifei92 Sep 15, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions examples/example-cluster.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,7 @@ spec:


## Configs that will be added to my.cnf for cluster
mysqlConf:
query-cache-type: "0"
query-cache-size: "0"
# mysqlConf:
# innodb-buffer-size: 128M


Expand Down
38 changes: 27 additions & 11 deletions hack/development/Dockerfile.sidecar
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,26 @@
# Copy the mysql-operator-sidecar into it's own image
# NOTE: this image is for development only
###############################################################################
# Build rclone
###############################################################################
FROM debian:stretch as rclone

RUN apt-get update \
&& apt-get install -y --no-install-recommends \
gnupg ca-certificates wget unzip

COPY hack/docker/rclone.gpg /root/rclone.gpg
RUN gpg --import /root/rclone.gpg

RUN wget -nv https://github.com/ncw/rclone/releases/download/v1.48.0/rclone-v1.48.0-linux-amd64.zip \
&& wget -nv https://github.com/ncw/rclone/releases/download/v1.48.0/SHA256SUMS \
&& gpg --verify --output=- SHA256SUMS > sums \
&& sha256sum -c --ignore-missing sums \
&& unzip rclone-*-linux-amd64.zip \
&& mv rclone-*-linux-amd64/rclone /usr/local/bin/ \
&& chmod 755 /usr/local/bin/rclone

###############################################################################
# Docker image for sidecar containers
###############################################################################
FROM debian:stretch-slim as sidecar

RUN groupadd -g 999 mysql
Expand All @@ -9,24 +30,19 @@ RUN useradd -u 999 -r -g 999 -s /sbin/nologin \
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
apt-transport-https ca-certificates wget \
gnupg1 dirmngr \
&& rm -rf /var/lib/apt/lists/*

RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 9334A25F8507EFA5

COPY hack/docker/percona.gpg /etc/apt/trusted.gpg.d/percona.gpg
RUN echo 'deb https://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list

RUN apt-get update \
&& apt-get install -y --no-install-recommends \
percona-toolkit percona-xtrabackup-24 unzip \
percona-toolkit percona-xtrabackup-80 unzip \
Copy link
Contributor

@AMecea AMecea Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same thing should be done for the root Dockerfile.sidecar

&& wget https://github.com/maxbube/mydumper/releases/download/v0.9.5/mydumper_0.9.5-2.stretch_amd64.deb \
&& dpkg -i mydumper_0.9.5-2.stretch_amd64.deb \
&& rm -rf mydumper_0.9.5-2.stretch_amd64.deb /var/lib/apt/lists/* \
&& wget https://downloads.rclone.org/rclone-current-linux-amd64.zip \
&& unzip rclone-current-linux-amd64.zip \
&& mv rclone-*-linux-amd64/rclone /usr/local/bin/ \
&& rm -rf rclone-*-linux-amd64 rclone-current-linux-amd64.zip \
&& chmod 755 /usr/local/bin/rclone
&& mkdir -p /usr/share/src \
&& wget https://github.com/maxbube/mydumper/archive/v0.9.5.tar.gz -O /usr/share/src/mydumper-v0.9.5.tar.gz

USER mysql

Expand Down
46 changes: 35 additions & 11 deletions pkg/controller/mysqlcluster/internal/syncer/config_map.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,15 @@ func buildMysqlConfData(cluster *mysqlcluster.MysqlCluster) (string, error) {
cfg := ini.Empty()
sec := cfg.Section("mysqld")

addKVConfigsToSection(sec, getMysqlMasterSlaveConfigs(), cluster.Spec.MysqlConf)
// common configs
addKVConfigsToSection(sec, convertMapToKVConfig(mysqlCommonConfigs), cluster.Spec.MysqlConf)
if cluster.GetMySQLSemVer().Major == 5 {
addKVConfigsToSection(sec, convertMapToKVConfig(mysql5xConfigs))
} else if cluster.GetMySQLSemVer().Major == 8 {
addKVConfigsToSection(sec, convertMapToKVConfig(mysql8xConfigs))
}

// boolean configs
addBConfigsToSection(sec, mysqlMasterSlaveBooleanConfigs)

// include configs from /etc/mysql/conf.d/*.cnf
Expand All @@ -85,10 +93,10 @@ func buildMysqlConfData(cluster *mysqlcluster.MysqlCluster) (string, error) {

}

func getMysqlMasterSlaveConfigs() map[string]intstr.IntOrString {
func convertMapToKVConfig(m map[string]string) map[string]intstr.IntOrString {
config := make(map[string]intstr.IntOrString)

for key, value := range mysqlMasterSlaveConfigs {
for key, value := range m {
config[key] = intstr.Parse(value)
}

Expand Down Expand Up @@ -142,8 +150,8 @@ func writeConfigs(cfg *ini.File) (string, error) {
return buf.String(), nil
}

// mysqlMasterSlaveConfigs represents the configuration that mysql-operator needs by default
var mysqlMasterSlaveConfigs = map[string]string{
// mysqlCommonConfigs represents the configuration that mysql-operator needs by default
var mysqlCommonConfigs = map[string]string{
"log-bin": "/var/lib/mysql/mysql-bin",
"log-slave-updates": "on",

Expand All @@ -168,14 +176,11 @@ var mysqlMasterSlaveConfigs = map[string]string{
// Safety
"max-allowed-packet": "16M",
"max-connect-errors": "1000000",
"sql-mode": "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER," +
"NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY",
"sysdate-is-now": "1",
"sysdate-is-now": "1",

// Binary logging
"expire-logs-days": "14",
"sync-binlog": "1",
"binlog-format": "ROW",
"sync-binlog": "1",
"binlog-format": "ROW",

// CACHES AND LIMITS
"tmp-table-size": "32M",
Expand All @@ -196,6 +201,25 @@ var mysqlMasterSlaveConfigs = map[string]string{
"collation-server": "utf8mb4_unicode_ci",
}

var mysql5xConfigs = map[string]string{
"query-cache-type": "0",
"query-cache-size": "0",
"sql-mode": "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER," +
"NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY",

"expire-logs-days": "14",
}

var mysql8xConfigs = map[string]string{
"sql-mode": "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION," +
"NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY",

"binlog_expire_logs_seconds": "1209600", // 14 days = 14 * 24 * 60 * 60

// use 5.7 auth plugin to be backward compatible
"default-authentication-plugin": "mysql_native_password",
}

var mysqlMasterSlaveBooleanConfigs = []string{
// Safety
"skip-name-resolve",
Expand Down
4 changes: 4 additions & 0 deletions pkg/controller/mysqlcluster/internal/syncer/statefullset.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,6 +210,10 @@ func (s *sfsSyncer) getEnvFor(name string) []core.EnvVar {
Name: "MY_FQDN",
Value: "$(MY_POD_NAME).$(MY_SERVICE_NAME).$(MY_NAMESPACE)",
})
env = append(env, core.EnvVar{
Name: "MY_MYSQL_VERSION",
Value: s.cluster.GetMySQLSemVer().String(),
})

if len(s.cluster.Spec.InitBucketURL) > 0 && name == containerCloneAndInitName {
env = append(env, core.EnvVar{
Expand Down
14 changes: 12 additions & 2 deletions pkg/sidecar/appconf.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,8 +165,13 @@ func initFileQuery(cfg *Config, gtidPurged string) []byte {
[]string{"SELECT", "CREATE"}, fmt.Sprintf("%s.%s", toolsDbName, toolsHeartbeatTableName))...)

// configure replication user
replPermissions := []string{"SELECT", "PROCESS", "RELOAD", "LOCK TABLES", "REPLICATION CLIENT", "REPLICATION SLAVE"}
if cfg.MySQLVersion.Major == 8 {
// if it's a mysql 8 then the backup user needs BACKUP_ADMIN permissions to take backups
replPermissions = append(replPermissions, "BACKUP_ADMIN")
}
queries = append(queries, createUserQuery(cfg.ReplicationUser, cfg.ReplicationPassword, "%",
[]string{"SELECT", "PROCESS", "RELOAD", "LOCK TABLES", "REPLICATION CLIENT", "REPLICATION SLAVE", "BACKUP_ADMIN"}, "*.*")...)
replPermissions, "*.*")...)

// configure metrics exporter user
queries = append(queries, createUserQuery(cfg.MetricsUser, cfg.MetricsPassword, "127.0.0.1",
Expand All @@ -186,10 +191,15 @@ func initFileQuery(cfg *Config, gtidPurged string) []byte {
// CSV engine for this table can't be used because we use REPLACE statement that requires PRIMARY KEY or
// UNIQUE KEY index
// nolint: gosec
if !strings.HasSuffix(cfg.Hostname, "-0") {
Copy link
Contributor

@AMecea AMecea Sep 15, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This requires some comments explaining why this is needed. I'm curious why this works? 😄

Also, there is a function that checks the pod ordinal:

Suggested change
if !strings.HasSuffix(cfg.Hostname, "-0") {
if !cfg.IsFirstPodInSet() {

queries = append(queries, fmt.Sprintf("DROP TABLE IF EXISTS %s.%s",
constants.OperatorDbName, constants.OperatorStatusTableName))
}

queries = append(queries, fmt.Sprintf(
"CREATE TABLE IF NOT EXISTS %[1]s.%[2]s ("+
" name varchar(64) PRIMARY KEY,"+
" value varchar(512) NOT NULL\n)",
" value varchar(512) NOT NULL )",
constants.OperatorDbName, constants.OperatorStatusTableName))

// mark node as not configured at startup, the operator will mark it configured
Expand Down
6 changes: 6 additions & 0 deletions pkg/sidecar/configs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
// add mysql driver
_ "github.com/go-sql-driver/mysql"

"github.com/blang/semver"
"github.com/presslabs/controller-util/rand"

"github.com/presslabs/mysql-operator/pkg/internal/mysqlcluster"
Expand Down Expand Up @@ -72,6 +73,9 @@ type Config struct {

// ExistsMySQLData checks if MySQL data is initialized by checking if the mysql dir exists
ExistsMySQLData bool

// MySQLVersion represents the current mysql version
MySQLVersion semver.Version
}

// FQDNForServer returns the pod hostname for given MySQL server id
Expand Down Expand Up @@ -151,6 +155,8 @@ func NewConfig() *Config {
HeartBeatPassword: hbPass,

ExistsMySQLData: eData,

MySQLVersion: semver.MustParse(getEnvValue("MY_MYSQL_VERSION")),
}

return cfg
Expand Down