From fdac9a991c5483181a612a13db2f590edfc6cce6 Mon Sep 17 00:00:00 2001 From: amecea Date: Thu, 12 Sep 2019 17:00:52 +0300 Subject: [PATCH 1/2] Fix MySQL 8.0 support --- examples/example-cluster.yaml | 4 +- hack/development/Dockerfile.sidecar | 38 ++++++++++----- .../internal/syncer/config_map.go | 46 ++++++++++++++----- .../internal/syncer/statefullset.go | 4 ++ pkg/sidecar/appconf.go | 9 +++- pkg/sidecar/configs.go | 6 +++ 6 files changed, 80 insertions(+), 27 deletions(-) diff --git a/examples/example-cluster.yaml b/examples/example-cluster.yaml index a702fcf5d..d552f86d2 100644 --- a/examples/example-cluster.yaml +++ b/examples/example-cluster.yaml @@ -21,9 +21,7 @@ spec: ## Configs that will be added to my.cnf for cluster - mysqlConf: - query-cache-type: "0" - query-cache-size: "0" + # mysqlConf: # innodb-buffer-size: 128M diff --git a/hack/development/Dockerfile.sidecar b/hack/development/Dockerfile.sidecar index b8d3511b8..af0030043 100644 --- a/hack/development/Dockerfile.sidecar +++ b/hack/development/Dockerfile.sidecar @@ -1,5 +1,26 @@ -# Copy the mysql-operator-sidecar into it's own image -# NOTE: this image is for development only +############################################################################### +# Build rclone +############################################################################### +FROM debian:stretch as rclone + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + gnupg ca-certificates wget unzip + +COPY hack/docker/rclone.gpg /root/rclone.gpg +RUN gpg --import /root/rclone.gpg + +RUN wget -nv https://github.com/ncw/rclone/releases/download/v1.48.0/rclone-v1.48.0-linux-amd64.zip \ + && wget -nv https://github.com/ncw/rclone/releases/download/v1.48.0/SHA256SUMS \ + && gpg --verify --output=- SHA256SUMS > sums \ + && sha256sum -c --ignore-missing sums \ + && unzip rclone-*-linux-amd64.zip \ + && mv rclone-*-linux-amd64/rclone /usr/local/bin/ \ + && chmod 755 /usr/local/bin/rclone + +############################################################################### +# Docker image for sidecar containers +############################################################################### FROM debian:stretch-slim as sidecar RUN groupadd -g 999 mysql @@ -9,24 +30,19 @@ RUN useradd -u 999 -r -g 999 -s /sbin/nologin \ RUN apt-get update \ && apt-get install -y --no-install-recommends \ apt-transport-https ca-certificates wget \ - gnupg1 dirmngr \ && rm -rf /var/lib/apt/lists/* -RUN apt-key adv --keyserver ha.pool.sks-keyservers.net --recv-keys 9334A25F8507EFA5 - +COPY hack/docker/percona.gpg /etc/apt/trusted.gpg.d/percona.gpg RUN echo 'deb https://repo.percona.com/apt stretch main' > /etc/apt/sources.list.d/percona.list RUN apt-get update \ && apt-get install -y --no-install-recommends \ - percona-toolkit percona-xtrabackup-24 unzip \ + percona-toolkit percona-xtrabackup-80 unzip \ && wget https://github.com/maxbube/mydumper/releases/download/v0.9.5/mydumper_0.9.5-2.stretch_amd64.deb \ && dpkg -i mydumper_0.9.5-2.stretch_amd64.deb \ && rm -rf mydumper_0.9.5-2.stretch_amd64.deb /var/lib/apt/lists/* \ - && wget https://downloads.rclone.org/rclone-current-linux-amd64.zip \ - && unzip rclone-current-linux-amd64.zip \ - && mv rclone-*-linux-amd64/rclone /usr/local/bin/ \ - && rm -rf rclone-*-linux-amd64 rclone-current-linux-amd64.zip \ - && chmod 755 /usr/local/bin/rclone + && mkdir -p /usr/share/src \ + && wget https://github.com/maxbube/mydumper/archive/v0.9.5.tar.gz -O /usr/share/src/mydumper-v0.9.5.tar.gz USER mysql diff --git a/pkg/controller/mysqlcluster/internal/syncer/config_map.go b/pkg/controller/mysqlcluster/internal/syncer/config_map.go index 0ff0e0b7e..0188d35ba 100644 --- a/pkg/controller/mysqlcluster/internal/syncer/config_map.go +++ b/pkg/controller/mysqlcluster/internal/syncer/config_map.go @@ -67,7 +67,15 @@ func buildMysqlConfData(cluster *mysqlcluster.MysqlCluster) (string, error) { cfg := ini.Empty() sec := cfg.Section("mysqld") - addKVConfigsToSection(sec, getMysqlMasterSlaveConfigs(), cluster.Spec.MysqlConf) + // common configs + addKVConfigsToSection(sec, convertMapToKVConfig(mysqlCommonConfigs), cluster.Spec.MysqlConf) + if cluster.GetMySQLSemVer().Major == 5 { + addKVConfigsToSection(sec, convertMapToKVConfig(mysql5xConfigs)) + } else if cluster.GetMySQLSemVer().Major == 8 { + addKVConfigsToSection(sec, convertMapToKVConfig(mysql8xConfigs)) + } + + // boolean configs addBConfigsToSection(sec, mysqlMasterSlaveBooleanConfigs) // include configs from /etc/mysql/conf.d/*.cnf @@ -85,10 +93,10 @@ func buildMysqlConfData(cluster *mysqlcluster.MysqlCluster) (string, error) { } -func getMysqlMasterSlaveConfigs() map[string]intstr.IntOrString { +func convertMapToKVConfig(m map[string]string) map[string]intstr.IntOrString { config := make(map[string]intstr.IntOrString) - for key, value := range mysqlMasterSlaveConfigs { + for key, value := range m { config[key] = intstr.Parse(value) } @@ -142,8 +150,8 @@ func writeConfigs(cfg *ini.File) (string, error) { return buf.String(), nil } -// mysqlMasterSlaveConfigs represents the configuration that mysql-operator needs by default -var mysqlMasterSlaveConfigs = map[string]string{ +// mysqlCommonConfigs represents the configuration that mysql-operator needs by default +var mysqlCommonConfigs = map[string]string{ "log-bin": "/var/lib/mysql/mysql-bin", "log-slave-updates": "on", @@ -168,14 +176,11 @@ var mysqlMasterSlaveConfigs = map[string]string{ // Safety "max-allowed-packet": "16M", "max-connect-errors": "1000000", - "sql-mode": "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER," + - "NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY", - "sysdate-is-now": "1", + "sysdate-is-now": "1", // Binary logging - "expire-logs-days": "14", - "sync-binlog": "1", - "binlog-format": "ROW", + "sync-binlog": "1", + "binlog-format": "ROW", // CACHES AND LIMITS "tmp-table-size": "32M", @@ -196,6 +201,25 @@ var mysqlMasterSlaveConfigs = map[string]string{ "collation-server": "utf8mb4_unicode_ci", } +var mysql5xConfigs = map[string]string{ + "query-cache-type": "0", + "query-cache-size": "0", + "sql-mode": "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER," + + "NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION,NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY", + + "expire-logs-days": "14", +} + +var mysql8xConfigs = map[string]string{ + "sql-mode": "STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_VALUE_ON_ZERO,NO_ENGINE_SUBSTITUTION," + + "NO_ZERO_DATE,NO_ZERO_IN_DATE,ONLY_FULL_GROUP_BY", + + "binlog_expire_logs_seconds": "1209600", // 14 days = 14 * 24 * 60 * 60 + + // use 5.7 auth plugin to be backward compatible + "default-authentication-plugin": "mysql_native_password", +} + var mysqlMasterSlaveBooleanConfigs = []string{ // Safety "skip-name-resolve", diff --git a/pkg/controller/mysqlcluster/internal/syncer/statefullset.go b/pkg/controller/mysqlcluster/internal/syncer/statefullset.go index df41a9dce..730492607 100644 --- a/pkg/controller/mysqlcluster/internal/syncer/statefullset.go +++ b/pkg/controller/mysqlcluster/internal/syncer/statefullset.go @@ -210,6 +210,10 @@ func (s *sfsSyncer) getEnvFor(name string) []core.EnvVar { Name: "MY_FQDN", Value: "$(MY_POD_NAME).$(MY_SERVICE_NAME).$(MY_NAMESPACE)", }) + env = append(env, core.EnvVar{ + Name: "MY_MYSQL_VERSION", + Value: s.cluster.GetMySQLSemVer().String(), + }) if len(s.cluster.Spec.InitBucketURL) > 0 && name == containerCloneAndInitName { env = append(env, core.EnvVar{ diff --git a/pkg/sidecar/appconf.go b/pkg/sidecar/appconf.go index f6e8613eb..0114f3c8d 100644 --- a/pkg/sidecar/appconf.go +++ b/pkg/sidecar/appconf.go @@ -165,8 +165,13 @@ func initFileQuery(cfg *Config, gtidPurged string) []byte { []string{"SELECT", "CREATE"}, fmt.Sprintf("%s.%s", toolsDbName, toolsHeartbeatTableName))...) // configure replication user + replPermissions := []string{"SELECT", "PROCESS", "RELOAD", "LOCK TABLES", "REPLICATION CLIENT", "REPLICATION SLAVE"} + if cfg.MySQLVersion.Major == 8 { + // if it's a mysql 8 then the backup user needs BACKUP_ADMIN permissions to take backups + replPermissions = append(replPermissions, "BACKUP_ADMIN") + } queries = append(queries, createUserQuery(cfg.ReplicationUser, cfg.ReplicationPassword, "%", - []string{"SELECT", "PROCESS", "RELOAD", "LOCK TABLES", "REPLICATION CLIENT", "REPLICATION SLAVE", "BACKUP_ADMIN"}, "*.*")...) + replPermissions, "*.*")...) // configure metrics exporter user queries = append(queries, createUserQuery(cfg.MetricsUser, cfg.MetricsPassword, "127.0.0.1", @@ -189,7 +194,7 @@ func initFileQuery(cfg *Config, gtidPurged string) []byte { queries = append(queries, fmt.Sprintf( "CREATE TABLE IF NOT EXISTS %[1]s.%[2]s ("+ " name varchar(64) PRIMARY KEY,"+ - " value varchar(512) NOT NULL\n)", + " value varchar(512) NOT NULL )", constants.OperatorDbName, constants.OperatorStatusTableName)) // mark node as not configured at startup, the operator will mark it configured diff --git a/pkg/sidecar/configs.go b/pkg/sidecar/configs.go index 81427df50..c72e1e637 100644 --- a/pkg/sidecar/configs.go +++ b/pkg/sidecar/configs.go @@ -27,6 +27,7 @@ import ( // add mysql driver _ "github.com/go-sql-driver/mysql" + "github.com/blang/semver" "github.com/presslabs/controller-util/rand" "github.com/presslabs/mysql-operator/pkg/internal/mysqlcluster" @@ -72,6 +73,9 @@ type Config struct { // ExistsMySQLData checks if MySQL data is initialized by checking if the mysql dir exists ExistsMySQLData bool + + // MySQLVersion represents the current mysql version + MySQLVersion semver.Version } // FQDNForServer returns the pod hostname for given MySQL server id @@ -151,6 +155,8 @@ func NewConfig() *Config { HeartBeatPassword: hbPass, ExistsMySQLData: eData, + + MySQLVersion: semver.MustParse(getEnvValue("MY_MYSQL_VERSION")), } return cfg From adfcaefa22be4aacaf83be91addd6139470e7e23 Mon Sep 17 00:00:00 2001 From: zhifei92 <13641019076@163.com> Date: Tue, 15 Sep 2020 15:00:11 +0800 Subject: [PATCH 2/2] fix:#363(file ./sys_operator/status.ibd is missing) --- pkg/sidecar/appconf.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/sidecar/appconf.go b/pkg/sidecar/appconf.go index 0114f3c8d..123c770a9 100644 --- a/pkg/sidecar/appconf.go +++ b/pkg/sidecar/appconf.go @@ -191,6 +191,11 @@ func initFileQuery(cfg *Config, gtidPurged string) []byte { // CSV engine for this table can't be used because we use REPLACE statement that requires PRIMARY KEY or // UNIQUE KEY index // nolint: gosec + if !strings.HasSuffix(cfg.Hostname, "-0") { + queries = append(queries, fmt.Sprintf("DROP TABLE IF EXISTS %s.%s", + constants.OperatorDbName, constants.OperatorStatusTableName)) + } + queries = append(queries, fmt.Sprintf( "CREATE TABLE IF NOT EXISTS %[1]s.%[2]s ("+ " name varchar(64) PRIMARY KEY,"+