From fb43ee92d60a5a03318bdd8cbc58f4701386eaa5 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Tue, 14 Jan 2020 15:36:30 +0100 Subject: [PATCH 01/61] WIP Connection pooler support Add an initial support for a connection pooler. The idea is to make it generic enough to be able to switch a corresponding docker image to change from pgbouncer to e.g. odyssey. Operator needs to create a deployment with pooler and a service for it to access. --- docker/DebugDockerfile | 13 +- manifests/minimal-postgres-manifest.yaml | 2 + manifests/operator-service-account-rbac.yaml | 1 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 12 ++ pkg/apis/acid.zalan.do/v1/register.go | 5 +- pkg/cluster/cluster.go | 55 ++++- pkg/cluster/database.go | 67 ++++++ pkg/cluster/k8sres.go | 216 ++++++++++++++++++- pkg/cluster/resources.go | 96 +++++++++ pkg/cluster/sync.go | 6 + pkg/cluster/types.go | 2 + pkg/cluster/util.go | 18 +- pkg/spec/types.go | 4 +- pkg/util/config/config.go | 11 + pkg/util/constants/roles.go | 23 +- pkg/util/k8sutil/k8sutil.go | 2 + 16 files changed, 513 insertions(+), 20 deletions(-) diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index 76dadf6df..0c11fe3b4 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -3,8 +3,17 @@ MAINTAINER Team ACID @ Zalando # We need root certificates to deal with teams api over https RUN apk --no-cache add ca-certificates go git musl-dev -RUN go get github.com/derekparker/delve/cmd/dlv COPY build/* / -CMD ["/root/go/bin/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] +RUN addgroup -g 1000 pgo +RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo + +RUN go get github.com/derekparker/delve/cmd/dlv +RUN cp /root/go/bin/dlv /dlv +RUN chown -R pgo:pgo /dlv + +USER pgo:pgo +RUN ls -l / + +CMD ["/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 75dfdf07f..ff7785cec 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -17,3 +17,5 @@ spec: foo: zalando # dbname: owner postgresql: version: "11" + connectionPool: + type: "pgbouncer" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index a37abe476..d7cd6fd74 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -119,6 +119,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 07b42d4d4..c50d9c902 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,6 +27,8 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` + ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -154,3 +156,13 @@ type UserFlags []string type PostgresStatus struct { PostgresClusterStatus string `json:"PostgresClusterStatus"` } + +// Options for connection pooler +type ConnectionPool struct { + NumberOfInstances *int32 `json:"instancesNumber,omitempty"` + Schema *string `json:"schema,omitempty"` + User *string `json:"user,omitempty"` + Type *string `json:"type,omitempty"` + Mode *string `json:"mode,omitempty"` + PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` +} diff --git a/pkg/apis/acid.zalan.do/v1/register.go b/pkg/apis/acid.zalan.do/v1/register.go index 1c30e35fb..34def209d 100644 --- a/pkg/apis/acid.zalan.do/v1/register.go +++ b/pkg/apis/acid.zalan.do/v1/register.go @@ -10,7 +10,8 @@ import ( // APIVersion of the `postgresql` and `operator` CRDs const ( - APIVersion = "v1" + APIVersion = "v1" + PostgresqlKind = "postgresql" ) var ( @@ -42,7 +43,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { // AddKnownType assumes derives the type kind from the type name, which is always uppercase. // For our CRDs we use lowercase names historically, therefore we have to supply the name separately. // TODO: User uppercase CRDResourceKind of our types in the next major API version - scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresql"), &Postgresql{}) + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind(PostgresqlKind), &Postgresql{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresqlList"), &PostgresqlList{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("OperatorConfiguration"), &OperatorConfiguration{}) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c560c4cdf..1681e7d2e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -48,11 +48,17 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding } +type ConnectionPoolResources struct { + Deployment *appsv1.Deployment + Service *v1.Service +} + type kubeResources struct { Services map[PostgresRole]*v1.Service Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet + ConnectionPool *ConnectionPoolResources PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -184,7 +190,8 @@ func (c *Cluster) isNewCluster() bool { func (c *Cluster) initUsers() error { c.setProcessName("initializing users") - // clear our the previous state of the cluster users (in case we are running a sync). + // clear our the previous state of the cluster users (in case we are + // running a sync). c.systemUsers = map[string]spec.PgUser{} c.pgUsers = map[string]spec.PgUser{} @@ -292,8 +299,10 @@ func (c *Cluster) Create() error { } c.logger.Infof("pods are ready") - // create database objects unless we are running without pods or disabled that feature explicitly + // create database objects unless we are running without pods or disabled + // that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { + c.logger.Infof("Create roles") if err = c.createRoles(); err != nil { return fmt.Errorf("could not create users: %v", err) } @@ -316,6 +325,26 @@ func (c *Cluster) Create() error { c.logger.Errorf("could not list resources: %v", err) } + // Create connection pool deployment and services if necessary. Since we + // need to peform some operations with the database itself (e.g. install + // lookup function), do it as the last step, when everything is available. + // + // Do not consider connection pool as a strict requirement, and if + // something fails, report warning + if c.needConnectionPool() { + if c.ConnectionPool != nil { + c.logger.Warning("Connection pool already exists in the cluster") + return nil + } + connPool, err := c.createConnectionPool() + if err != nil { + c.logger.Warningf("could not create connection pool: %v", err) + return nil + } + c.logger.Infof("connection pool %q has been successfully created", + util.NameFromMeta(connPool.Deployment.ObjectMeta)) + } + return nil } @@ -745,6 +774,12 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not remove leftover patroni objects; %v", err) } + // Delete connection pool objects anyway, even if it's not mentioned in the + // manifest, just to not keep orphaned components in case if something went + // wrong + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } } //NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). @@ -811,6 +846,22 @@ func (c *Cluster) initSystemUsers() { Name: c.OpConfig.ReplicationUsername, Password: util.RandomPassword(constants.PasswordLength), } + + // Connection pool user is an exception, if requested it's going to be + // created by operator as a normal pgUser + if c.needConnectionPool() { + + username := c.Spec.ConnectionPool.User + if username == nil { + username = &c.OpConfig.ConnectionPool.User + } + + c.systemUsers[constants.ConnectionPoolUserKeyName] = spec.PgUser{ + Origin: spec.RoleConnectionPool, + Name: *username, + Password: util.RandomPassword(constants.PasswordLength), + } + } } func (c *Cluster) initRobotUsers() error { diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 07ea011a6..1b74bd6b6 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -1,10 +1,12 @@ package cluster import ( + "bytes" "database/sql" "fmt" "net" "strings" + "text/template" "time" "github.com/lib/pq" @@ -28,6 +30,25 @@ const ( getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + connectionPoolLookup = ` + CREATE SCHEMA IF NOT EXISTS {{.pool_schema}}; + + CREATE OR REPLACE FUNCTION {{.pool_schema}}.user_lookup( + in i_username text, out uname text, out phash text) + RETURNS record AS $$ + BEGIN + SELECT usename, passwd FROM pg_catalog.pg_shadow + WHERE usename = i_username INTO uname, phash; + RETURN; + END; + $$ LANGUAGE plpgsql SECURITY DEFINER; + + REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) + FROM public, {{.pool_schema}}; + GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) + TO {{.pool_user}}; + GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; + ` ) func (c *Cluster) pgConnectionString() string { @@ -243,3 +264,49 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } + +// Creates a connection pool credentials lookup function in every database to +// perform remote authentification. +func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { + var stmtBytes bytes.Buffer + + if err := c.initDbConn(); err != nil { + return fmt.Errorf("could not init database connection") + } + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + currentDatabases, err := c.getDatabases() + if err != nil { + msg := "could not get databases to install pool lookup function: %v" + return fmt.Errorf(msg, err) + } + + templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) + + for dbname, _ := range currentDatabases { + c.logger.Infof("Install pool lookup function into %s", dbname) + + params := TemplateParams{ + "pool_schema": poolSchema, + "pool_user": poolUser, + } + + if err := templater.Execute(&stmtBytes, params); err != nil { + return fmt.Errorf("could not prepare sql statement %+v: %v", + params, err) + } + + if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + return fmt.Errorf("could not execute sql statement %s: %v", + stmtBytes.String(), err) + } + + c.logger.Infof("Pool lookup function installed into %s", dbname) + } + + return nil +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e6561e0f3..7d6b9be07 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -31,6 +31,8 @@ const ( patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" localHost = "127.0.0.1/32" + connectionPoolContainer = "connection-pool" + pgPort = 5432 ) type pgUser struct { @@ -66,6 +68,10 @@ func (c *Cluster) statefulSetName() string { return c.Name } +func (c *Cluster) connPoolName() string { + return c.Name + "-pooler" +} + func (c *Cluster) endpointName(role PostgresRole) string { name := c.Name if role == Replica { @@ -84,6 +90,28 @@ func (c *Cluster) serviceName(role PostgresRole) string { return name } +func (c *Cluster) serviceAddress(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return service.ObjectMeta.Name + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + +func (c *Cluster) servicePort(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return fmt.Sprint(service.Spec.Ports[0].Port) + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + func (c *Cluster) podDisruptionBudgetName() string { return c.OpConfig.PDBNameFormat.Format("cluster", c.Name) } @@ -315,7 +343,11 @@ func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]stri return *tolerationsSpec } - if len(podToleration["key"]) > 0 || len(podToleration["operator"]) > 0 || len(podToleration["value"]) > 0 || len(podToleration["effect"]) > 0 { + if len(podToleration["key"]) > 0 || + len(podToleration["operator"]) > 0 || + len(podToleration["value"]) > 0 || + len(podToleration["effect"]) > 0 { + return []v1.Toleration{ { Key: podToleration["key"], @@ -1669,3 +1701,185 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } + +func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( + *v1.PodTemplateSpec, error) { + + podTemplate := spec.ConnectionPool.PodTemplate + + if podTemplate == nil { + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + c.Spec.Resources, + c.makeDefaultResources()) + + effectiveMode := spec.ConnectionPool.Mode + if effectiveMode == nil { + effectiveMode = &c.OpConfig.ConnectionPool.Mode + } + + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } + + secretSelector := func(key string) *v1.SecretKeySelector { + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(c.OpConfig.SuperUsername), + }, + Key: key, + } + } + + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(Master), + }, + { + Name: "PGPORT", + Value: c.servicePort(Master), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + // the convention is to use the same schema name as + // connection pool username + { + Name: "PGSCHEMA", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), + }, + }, + { + Name: "CONNECTION_POOL_MODE", + Value: *effectiveMode, + }, + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + } + + poolerContainer := v1.Container{ + Name: connectionPoolContainer, + Image: c.OpConfig.ConnectionPool.Image, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, + }, + }, + Env: envVars, + } + + podTemplate = &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connPoolLabelsSelector().MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, + } + } + + return podTemplate, nil +} + +func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( + *appsv1.Deployment, error) { + + podTemplate, err := c.generateConnPoolPodTemplate(spec) + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances + } + + if err != nil { + return nil, err + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: map[string]string{}, + // make Postgresql CRD object its owner, so that if CRD object is + // deleted, this object will be deleted even if something went + // wrong and operator didn't deleted it. + OwnerReferences: []metav1.OwnerReference{ + { + UID: c.Statefulset.ObjectMeta.UID, + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: c.Statefulset.ObjectMeta.Name, + }, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: numberOfInstances, + Selector: c.connPoolLabelsSelector(), + Template: *podTemplate, + }, + } + + return deployment, nil +} + +func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { + serviceSpec := v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: c.connPoolName(), + Port: pgPort, + TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, + }, + }, + Type: v1.ServiceTypeClusterIP, + Selector: map[string]string{ + "connection-pool": c.connPoolName(), + }, + } + + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: map[string]string{}, + // make Postgresql CRD object its owner, so that if CRD object is + // deleted, this object will be deleted even if something went + // wrong and operator didn't deleted it. + OwnerReferences: []metav1.OwnerReference{ + { + UID: c.Postgresql.ObjectMeta.UID, + APIVersion: acidv1.APIVersion, + Kind: acidv1.PostgresqlKind, + Name: c.Postgresql.ObjectMeta.Name, + }, + }, + }, + Spec: serviceSpec, + } + + return service +} diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c94a7bb46..7baa96c02 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -90,6 +90,102 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } +// Prepare the database for connection pool to be used, i.e. install lookup +// function (do it first, because it should be fast and if it didn't succeed, +// it doesn't makes sense to create more K8S objects. At this moment we assume +// that necessary connection pool user exists. +// +// After that create all the objects for connection pool, namely a deployment +// with a chosen pooler and a service to expose it. +func (c *Cluster) createConnectionPool() (*ConnectionPoolResources, error) { + var msg string + c.setProcessName("creating connection pool") + + err := c.installLookupFunction( + c.OpConfig.ConnectionPool.Schema, + c.OpConfig.ConnectionPool.User) + + if err != nil { + msg = "could not prepare database for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + deploymentSpec, err := c.generateConnPoolDeployment(&c.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return nil, err + } + + serviceSpec := c.generateConnPoolService(&c.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return nil, err + } + + c.ConnectionPool = &ConnectionPoolResources{ + Deployment: deployment, + Service: service, + } + c.logger.Debugf("created new connection pool %q, uid: %q", + util.NameFromMeta(deployment.ObjectMeta), deployment.UID) + + return c.ConnectionPool, nil +} + +func (c *Cluster) deleteConnectionPool() (err error) { + c.setProcessName("deleting connection pool") + c.logger.Debugln("deleting connection pool") + + // Lack of connection pooler objects is not a fatal error, just log it if + // it was present before in the manifest + if c.needConnectionPool() && c.ConnectionPool == nil { + c.logger.Infof("No connection pool to delete") + return nil + } + + deployment := c.ConnectionPool.Deployment + err = c.KubeClient. + Deployments(deployment.Namespace). + Delete(deployment.Name, c.deleteOptions) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool deployment was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete deployment: %v", err) + } + + c.logger.Infof("Connection pool deployment %q has been deleted", + util.NameFromMeta(deployment.ObjectMeta)) + + service := c.ConnectionPool.Service + err = c.KubeClient. + Services(service.Namespace). + Delete(service.Name, c.deleteOptions) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool service was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete service: %v", err) + } + + c.logger.Infof("Connection pool service %q has been deleted", + util.NameFromMeta(deployment.ObjectMeta)) + + c.ConnectionPool = nil + return nil +} + func getPodIndex(podName string) (int32, error) { parts := strings.Split(podName, "-") if len(parts) == 0 { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index fa4fc9ec1..5ee827d4b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -456,6 +456,12 @@ func (c *Cluster) syncRoles() (err error) { for _, u := range c.pgUsers { userNames = append(userNames, u.Name) } + + // An exception from system users, connection pool user + connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] + userNames = append(userNames, connPoolUser.Name) + c.pgUsers[connPoolUser.Name] = connPoolUser + dbUsers, err = c.readPgUsersFromDatabase(userNames) if err != nil { return fmt.Errorf("error getting users from the database: %v", err) diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 138b7015c..286505621 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -69,3 +69,5 @@ type ClusterStatus struct { Spec acidv1.PostgresSpec Error error } + +type TemplateParams map[string]interface{} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 8c02fed2e..d5f9c744f 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -408,7 +408,19 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set { } func (c *Cluster) labelsSelector() *metav1.LabelSelector { - return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil} + return &metav1.LabelSelector{ + MatchLabels: c.labelsSet(false), + MatchExpressions: nil, + } +} + +func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "connection-pool": c.connPoolName(), + }, + MatchExpressions: nil, + } } func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set { @@ -483,3 +495,7 @@ func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) { func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } + +func (c *Cluster) needConnectionPool() bool { + return c.Spec.ConnectionPool != nil +} diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 3e6bec8db..6f071c44a 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -23,13 +23,15 @@ const fileWithNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespa // RoleOrigin contains the code of the origin of a role type RoleOrigin int -// The rolesOrigin constant values must be sorted by the role priority for resolveNameConflict(...) to work. +// The rolesOrigin constant values must be sorted by the role priority for +// resolveNameConflict(...) to work. const ( RoleOriginUnknown RoleOrigin = iota RoleOriginManifest RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem + RoleConnectionPool ) type syncUserOperation int diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e4e429abb..a7a522566 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -83,6 +83,16 @@ type LogicalBackup struct { LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` } +// Operator options for connection pooler +type ConnectionPool struct { + NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` + Schema string `name:"connection_pool_schema" default:"pooler"` + User string `name:"connection_pool_user" default:"pooler"` + Type string `name:"connection_pool_type" default:"pgbouncer"` + Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` + Mode string `name:"connection_pool_mode" default:"session"` +} + // Config describes operator config type Config struct { CRD @@ -90,6 +100,7 @@ type Config struct { Auth Scalyr LogicalBackup + ConnectionPool WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 2c20d69db..3d201142c 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -2,15 +2,16 @@ package constants // Roles specific constants const ( - PasswordLength = 64 - SuperuserKeyName = "superuser" - ReplicationUserKeyName = "replication" - RoleFlagSuperuser = "SUPERUSER" - RoleFlagInherit = "INHERIT" - RoleFlagLogin = "LOGIN" - RoleFlagNoLogin = "NOLOGIN" - RoleFlagCreateRole = "CREATEROLE" - RoleFlagCreateDB = "CREATEDB" - RoleFlagReplication = "REPLICATION" - RoleFlagByPassRLS = "BYPASSRLS" + PasswordLength = 64 + SuperuserKeyName = "superuser" + ConnectionPoolUserKeyName = "pooler" + ReplicationUserKeyName = "replication" + RoleFlagSuperuser = "SUPERUSER" + RoleFlagInherit = "INHERIT" + RoleFlagLogin = "LOGIN" + RoleFlagNoLogin = "NOLOGIN" + RoleFlagCreateRole = "CREATEROLE" + RoleFlagCreateDB = "CREATEDB" + RoleFlagReplication = "REPLICATION" + RoleFlagByPassRLS = "BYPASSRLS" ) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 118d1df53..672d94634 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -40,6 +40,7 @@ type KubernetesClient struct { corev1.NamespacesGetter corev1.ServiceAccountsGetter appsv1.StatefulSetsGetter + appsv1.DeploymentsGetter rbacv1beta1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter apiextbeta1.CustomResourceDefinitionsGetter @@ -102,6 +103,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.NodesGetter = client.CoreV1() kubeClient.NamespacesGetter = client.CoreV1() kubeClient.StatefulSetsGetter = client.AppsV1() + kubeClient.DeploymentsGetter = client.AppsV1() kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1beta1() From 4c69b2b996d47c00692020563fd3038084bafb9a Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 15 Jan 2020 16:46:03 +0100 Subject: [PATCH 02/61] Improve cleaning up Set up a proper owner reference to StatefulSet, and delete with foreground policy to not leave orphans. --- pkg/cluster/k8sres.go | 56 ++++++++++++++++++++++++---------------- pkg/cluster/resources.go | 10 +++++-- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 7d6b9be07..209402a20 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1804,6 +1804,26 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( return podTemplate, nil } +// Return an array of ownerReferences to make an arbitraty object dependent on +// the StatefulSet. Dependency is made on StatefulSet instead of PostgreSQL CRD +// while the former is represent the actual state, and only it's deletion means +// we delete the cluster (e.g. if CRD was deleted, StatefulSet somehow +// survived, we can't delete an object because it will affect the functioning +// cluster). +func (c *Cluster) ownerReferences() []metav1.OwnerReference { + controller := true + + return []metav1.OwnerReference{ + { + UID: c.Statefulset.ObjectMeta.UID, + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: c.Statefulset.ObjectMeta.Name, + Controller: &controller, + }, + } +} + func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { @@ -1823,17 +1843,13 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( Namespace: c.Namespace, Labels: c.labelsSet(true), Annotations: map[string]string{}, - // make Postgresql CRD object its owner, so that if CRD object is - // deleted, this object will be deleted even if something went - // wrong and operator didn't deleted it. - OwnerReferences: []metav1.OwnerReference{ - { - UID: c.Statefulset.ObjectMeta.UID, - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: c.Statefulset.ObjectMeta.Name, - }, - }, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Ophaned" + // propagation policy, which means that it's deletion will not + // clean up this deployment, but there is a hope that this object + // will be garbage collected if something went wrong and operator + // didn't deleted it. + OwnerReferences: c.ownerReferences(), }, Spec: appsv1.DeploymentSpec{ Replicas: numberOfInstances, @@ -1866,17 +1882,13 @@ func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service Namespace: c.Namespace, Labels: c.labelsSet(true), Annotations: map[string]string{}, - // make Postgresql CRD object its owner, so that if CRD object is - // deleted, this object will be deleted even if something went - // wrong and operator didn't deleted it. - OwnerReferences: []metav1.OwnerReference{ - { - UID: c.Postgresql.ObjectMeta.UID, - APIVersion: acidv1.APIVersion, - Kind: acidv1.PostgresqlKind, - Name: c.Postgresql.ObjectMeta.Name, - }, - }, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Ophaned" + // propagation policy, which means that it's deletion will not + // clean up this service, but there is a hope that this object will + // be garbage collected if something went wrong and operator didn't + // deleted it. + OwnerReferences: c.ownerReferences(), }, Spec: serviceSpec, } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 7baa96c02..b4c7e578f 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -154,10 +154,14 @@ func (c *Cluster) deleteConnectionPool() (err error) { return nil } + // set delete propagation policy to foreground, so that replica set will be + // also deleted. + policy := metav1.DeletePropagationForeground + options := metav1.DeleteOptions{PropagationPolicy: &policy} deployment := c.ConnectionPool.Deployment err = c.KubeClient. Deployments(deployment.Namespace). - Delete(deployment.Name, c.deleteOptions) + Delete(deployment.Name, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool deployment was already deleted") @@ -168,10 +172,12 @@ func (c *Cluster) deleteConnectionPool() (err error) { c.logger.Infof("Connection pool deployment %q has been deleted", util.NameFromMeta(deployment.ObjectMeta)) + // set delete propagation policy to foreground, so that all the dependant + // will be deleted. service := c.ConnectionPool.Service err = c.KubeClient. Services(service.Namespace). - Delete(service.Name, c.deleteOptions) + Delete(service.Name, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool service was already deleted") From 2b2f29ff0b8b9f7c56cbac582bd90405be2d4c84 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 20 Jan 2020 16:06:45 +0100 Subject: [PATCH 03/61] Add CRD configuration With convertion for config, and start tests. --- .../v1/operator_configuration_type.go | 15 +++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 + pkg/cluster/k8sres.go | 47 +++++++++++--- pkg/cluster/k8sres_test.go | 65 +++++++++++++++++++ pkg/controller/operator_config.go | 12 ++++ pkg/util/config/config.go | 16 +++-- 6 files changed, 142 insertions(+), 15 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ded5261fb..58f171843 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -152,6 +152,20 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } +// Defines default configuration for connection pool +type ConnectionPoolConfiguration struct { + NumberOfInstances *int32 `json:"connection_pool_instances_number,omitempty"` + Schema string `json:"connection_pool_schema,omitempty"` + User string `json:"connection_pool_user,omitempty"` + Type string `json:"connection_pool_type,omitempty"` + Image string `json:"connection_pool_image,omitempty"` + Mode string `json:"connection_pool_mode,omitempty"` + DefaultCPURequest string `name:"connection_pool_default_cpu_request,omitempty"` + DefaultMemoryRequest string `name:"connection_pool_default_memory_request,omitempty"` + DefaultCPULimit string `name:"connection_pool_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `name:"connection_pool_default_memory_limit,omitempty"` +} + // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { Schedule string `json:"logical_backup_schedule,omitempty"` @@ -188,6 +202,7 @@ type OperatorConfigurationData struct { LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` Scalyr ScalyrConfiguration `json:"scalyr"` LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` + ConnectionPool ConnectionPoolConfiguration `json:"connection_pool"` } //Duration shortens this frequently used name diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index c50d9c902..e4d56c6e8 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -165,4 +165,6 @@ type ConnectionPool struct { Type *string `json:"type,omitempty"` Mode *string `json:"mode,omitempty"` PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` + + Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 209402a20..f992c2244 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -120,10 +120,39 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { config := c.OpConfig - defaultRequests := acidv1.ResourceDescription{CPU: config.DefaultCPURequest, Memory: config.DefaultMemoryRequest} - defaultLimits := acidv1.ResourceDescription{CPU: config.DefaultCPULimit, Memory: config.DefaultMemoryLimit} + defaultRequests := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPURequest, + Memory: config.Resources.DefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPULimit, + Memory: config.Resources.DefaultMemoryLimit, + } - return acidv1.Resources{ResourceRequests: defaultRequests, ResourceLimits: defaultLimits} + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } +} + +// Generate default resource section for connection pool deployment, to be used +// if nothing custom is specified in the manifest +func (c *Cluster) makeDefaultConnPoolResources() acidv1.Resources { + config := c.OpConfig + + defaultRequests := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPURequest, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPULimit, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } } func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { @@ -765,12 +794,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef request := spec.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } limit := spec.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(request, limit) @@ -792,12 +821,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // TODO #413 sidecarRequest := sidecar.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } sidecarLimit := sidecar.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) @@ -1710,8 +1739,8 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( if podTemplate == nil { gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) resources, err := generateResourceRequirements( - c.Spec.Resources, - c.makeDefaultResources()) + spec.ConnectionPool.Resources, + c.makeDefaultConnPoolResources()) effectiveMode := spec.ConnectionPool.Mode if effectiveMode == nil { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e8fe05456..aa9ef6513 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,7 @@ package cluster import ( + "errors" "reflect" v1 "k8s.io/api/core/v1" @@ -451,3 +452,67 @@ func TestSecretVolume(t *testing.T) { } } } + +func TestConnPoolPodTemplate(t *testing.T) { + testName := "Test connection pool pod template generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + var clusterNoDefaultRes = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{}, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + }{ + { + subTest: "empty pod template", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + }, + { + subTest: "no default resources", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), + cluster: clusterNoDefaultRes, + }, + } + for _, tt := range tests { + _, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + } +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c6f10faa0..f5d280363 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -142,5 +142,17 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit + // connection pool + result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances + result.ConnectionPool.Schema = fromCRD.ConnectionPool.Schema + result.ConnectionPool.User = fromCRD.ConnectionPool.User + result.ConnectionPool.Type = fromCRD.ConnectionPool.Type + result.ConnectionPool.Image = fromCRD.ConnectionPool.Image + result.ConnectionPool.Mode = fromCRD.ConnectionPool.Mode + result.ConnectionPool.ConnPoolDefaultCPURequest = fromCRD.ConnectionPool.DefaultCPURequest + result.ConnectionPool.ConnPoolDefaultMemoryRequest = fromCRD.ConnectionPool.DefaultMemoryRequest + result.ConnectionPool.ConnPoolDefaultCPULimit = fromCRD.ConnectionPool.DefaultCPULimit + result.ConnectionPool.ConnPoolDefaultMemoryLimit = fromCRD.ConnectionPool.DefaultMemoryLimit + return result } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index a7a522566..2baf99931 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,12 +85,16 @@ type LogicalBackup struct { // Operator options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` - Schema string `name:"connection_pool_schema" default:"pooler"` - User string `name:"connection_pool_user" default:"pooler"` - Type string `name:"connection_pool_type" default:"pgbouncer"` - Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` - Mode string `name:"connection_pool_mode" default:"session"` + NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` + Schema string `name:"connection_pool_schema" default:"pooler"` + User string `name:"connection_pool_user" default:"pooler"` + Type string `name:"connection_pool_type" default:"pgbouncer"` + Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` + Mode string `name:"connection_pool_mode" default:"session"` + ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"100m"` + ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` + ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"3"` + ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"1Gi"` } // Config describes operator config From b40ea2c4263fbca4c0521b87e5df11fdd1aec512 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 22 Jan 2020 14:49:54 +0100 Subject: [PATCH 04/61] Add more tests --- pkg/cluster/cluster.go | 2 +- pkg/cluster/k8sres.go | 5 + pkg/cluster/k8sres_test.go | 327 +++++++++++++++++++++++++++++++++- pkg/cluster/resources.go | 4 +- pkg/cluster/resources_test.go | 65 +++++++ pkg/cluster/types.go | 2 + pkg/util/k8sutil/k8sutil.go | 57 +++++- 7 files changed, 452 insertions(+), 10 deletions(-) create mode 100644 pkg/cluster/resources_test.go diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1681e7d2e..afcb0df82 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -336,7 +336,7 @@ func (c *Cluster) Create() error { c.logger.Warning("Connection pool already exists in the cluster") return nil } - connPool, err := c.createConnectionPool() + connPool, err := c.createConnectionPool(c.installLookupFunction) if err != nil { c.logger.Warningf("could not create connection pool: %v", err) return nil diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index f992c2244..8b7860886 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1842,6 +1842,11 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( func (c *Cluster) ownerReferences() []metav1.OwnerReference { controller := true + if c.Statefulset == nil { + c.logger.Warning("Cannot get owner reference, no statefulset") + return []metav1.OwnerReference{} + } + return []metav1.OwnerReference{ { UID: c.Statefulset.ObjectMeta.UID, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index aa9ef6513..012df4072 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -2,6 +2,7 @@ package cluster import ( "errors" + "fmt" "reflect" v1 "k8s.io/api/core/v1" @@ -14,6 +15,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + appsv1 "k8s.io/api/apps/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -453,7 +455,80 @@ func TestSecretVolume(t *testing.T) { } } -func TestConnPoolPodTemplate(t *testing.T) { +func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] + if cpuReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest { + return fmt.Errorf("CPU request doesn't match, got %s, expected %s", + cpuReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest) + } + + memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] + if memReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest { + return fmt.Errorf("Memory request doesn't match, got %s, expected %s", + memReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest) + } + + cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] + if cpuLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit { + return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", + cpuLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit) + } + + memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] + if memLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit { + return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", + memLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit) + } + + return nil +} + +func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + poolLabels := podSpec.ObjectMeta.Labels["connection-pool"] + + if poolLabels != cluster.connPoolLabelsSelector().MatchLabels["connection-pool"] { + return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", + podSpec.ObjectMeta.Labels, cluster.connPoolLabelsSelector().MatchLabels) + } + + return nil +} + +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + required := map[string]bool{ + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOL_MODE": false, + "CONNECTION_POOL_PORT": false, + } + + envs := podSpec.Spec.Containers[0].Env + for _, env := range envs { + required[env.Name] = true + } + + for env, value := range required { + if !value { + return fmt.Errorf("Environment variable %s is not present", env) + } + } + + return nil +} + +func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + if podSpec.ObjectMeta.Name != "test-pod-template" { + return fmt.Errorf("Custom pod template is not used, current spec %+v", + podSpec) + } + + return nil +} + +func TestConnPoolPodSpec(t *testing.T) { testName := "Test connection pool pod template generation" var cluster = New( Config{ @@ -484,19 +559,23 @@ func TestConnPoolPodTemplate(t *testing.T) { }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } + tests := []struct { subTest string spec *acidv1.PostgresSpec expected error cluster *Cluster + check func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error }{ { - subTest: "empty pod template", + subTest: "default configuration", spec: &acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{}, }, expected: nil, cluster: cluster, + check: noCheck, }, { subTest: "no default resources", @@ -505,14 +584,256 @@ func TestConnPoolPodTemplate(t *testing.T) { }, expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), cluster: clusterNoDefaultRes, + check: noCheck, + }, + { + subTest: "default resources are set", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testResources, + }, + { + subTest: "labels for service", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testLabels, + }, + { + subTest: "required envs", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testEnvs, + }, + { + subTest: "custom pod template", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + PodTemplate: &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-template", + }, + }, + }, + }, + expected: nil, + cluster: cluster, + check: testCustomPodTemplate, }, } for _, tt := range tests { - _, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) if err != tt.expected && err.Error() != tt.expected.Error() { t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", testName, tt.subTest, err, tt.expected) } + + err = tt.check(cluster, podSpec) + if err != nil { + t.Errorf("%s [%s]: Pod spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployment) error { + owner := deployment.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { + labels := deployment.Spec.Selector.MatchLabels + expected := cluster.connPoolLabelsSelector().MatchLabels + + if labels["connection-pool"] != expected["connection-pool"] { + return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", + labels, expected) + } + + return nil +} + +func TestConnPoolDeploymentSpec(t *testing.T) { + testName := "Test connection pool deployment spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *appsv1.Deployment) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, deployment *appsv1.Deployment) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testDeploymentOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testSelector, + }, + } + for _, tt := range tests { + deployment, err := tt.cluster.generateConnPoolDeployment(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, deployment) + if err != nil { + t.Errorf("%s [%s]: Deployment spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { + owner := service.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testServiceSelector(cluster *Cluster, service *v1.Service) error { + selector := service.Spec.Selector + + if selector["connection-pool"] != cluster.connPoolName() { + return fmt.Errorf("Selector is incorrect, got %s, expected %s", + selector["connection-pool"], cluster.connPoolName()) + } + + return nil +} + +func TestConnPoolServiceSpec(t *testing.T) { + testName := "Test connection pool service spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *v1.Service) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + check func(cluster *Cluster, deployment *v1.Service) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceSelector, + }, + } + for _, tt := range tests { + service := tt.cluster.generateConnPoolService(tt.spec) + + if err := tt.check(cluster, service); err != nil { + t.Errorf("%s [%s]: Service spec is incorrect, %+v", + testName, tt.subTest, err) + } } } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index b4c7e578f..4f9d72e19 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -97,11 +97,11 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { // // After that create all the objects for connection pool, namely a deployment // with a chosen pooler and a service to expose it. -func (c *Cluster) createConnectionPool() (*ConnectionPoolResources, error) { +func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolResources, error) { var msg string c.setProcessName("creating connection pool") - err := c.installLookupFunction( + err := lookup( c.OpConfig.ConnectionPool.Schema, c.OpConfig.ConnectionPool.User) diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go new file mode 100644 index 000000000..a3754c564 --- /dev/null +++ b/pkg/cluster/resources_test.go @@ -0,0 +1,65 @@ +package cluster + +import ( + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mockInstallLookupFunction(schema string, user string) error { + return nil +} + +func TestConnPoolCreationAndDeletion(t *testing.T) { + testName := "Test connection pool creation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + poolResources, err := cluster.createConnectionPool(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pool, %s, %+v", + testName, err, poolResources) + } + + if poolResources.Deployment == nil { + t.Errorf("%s: Connection pool deployment is empty", testName) + } + + if poolResources.Service == nil { + t.Errorf("%s: Connection pool service is empty", testName) + } + + err = cluster.deleteConnectionPool() + if err != nil { + t.Errorf("%s: Cannot delete connection pool, %s", testName, err) + } +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 286505621..04d00cb58 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -71,3 +71,5 @@ type ClusterStatus struct { } type TemplateParams map[string]interface{} + +type InstallFunction func(schema string, user string) error diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 672d94634..77e46476b 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -10,6 +10,7 @@ import ( clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" "github.com/zalando/postgres-operator/pkg/util/constants" + apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -57,6 +58,20 @@ type mockSecret struct { type MockSecretGetter struct { } +type mockDeployment struct { + appsv1.DeploymentInterface +} + +type MockDeploymentGetter struct { +} + +type mockService struct { + corev1.ServiceInterface +} + +type MockServiceGetter struct { +} + type mockConfigMap struct { corev1.ConfigMapInterface } @@ -217,19 +232,53 @@ func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigM } // Secrets to be mocked -func (c *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { +func (mock *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { return &mockSecret{} } // ConfigMaps to be mocked -func (c *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { +func (mock *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { return &mockConfigMap{} } +func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeployment{} +} + +func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + +func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface { + return &mockService{} +} + +func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + // NewMockKubernetesClient for other tests func NewMockKubernetesClient() KubernetesClient { return KubernetesClient{ - SecretsGetter: &MockSecretGetter{}, - ConfigMapsGetter: &MockConfigMapsGetter{}, + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + DeploymentsGetter: &MockDeploymentGetter{}, + ServicesGetter: &MockServiceGetter{}, } } From 6c3752068bc7d5c13e5127b2b474584761eadffb Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 12 Feb 2020 17:28:48 +0100 Subject: [PATCH 05/61] Various improvements Add synchronization logic. For now get rid of podTemplate, type fields. Add crd validation & configuration part, put retry on top of lookup function installation. --- .../templates/clusterrole.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 37 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 18 +- pkg/cluster/cluster.go | 44 ++++- pkg/cluster/database.go | 22 ++- pkg/cluster/k8sres.go | 163 +++++++++--------- pkg/cluster/k8sres_test.go | 15 -- pkg/cluster/resources.go | 75 +++++++- pkg/cluster/sync.go | 111 ++++++++++++ pkg/cluster/sync_test.go | 125 ++++++++++++++ pkg/cluster/util.go | 2 +- pkg/controller/operator_config.go | 57 ++++-- pkg/util/constants/pooler.go | 13 ++ pkg/util/k8sutil/k8sutil.go | 93 ++++++++++ 14 files changed, 647 insertions(+), 129 deletions(-) create mode 100644 pkg/cluster/sync_test.go create mode 100644 pkg/util/constants/pooler.go diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index f8550a539..316f7de15 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -106,6 +106,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 7bd5c529c..c44955771 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -294,6 +294,43 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" status: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index e4d56c6e8..c56d70626 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,7 +27,8 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + EnableConnectionPool bool `json:"enable_connection_pool,omitempty"` + ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -159,12 +160,15 @@ type PostgresStatus struct { // Options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `json:"instancesNumber,omitempty"` - Schema *string `json:"schema,omitempty"` - User *string `json:"user,omitempty"` - Type *string `json:"type,omitempty"` - Mode *string `json:"mode,omitempty"` - PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` + NumberOfInstances *int32 `json:"instancesNumber,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` + // TODO: prepared snippets of configuration, one can choose via type, e.g. + // pgbouncer-large (with higher resources) or odyssey-small (with smaller + // resources) + // Type string `json:"type,omitempty"` Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index afcb0df82..8e65b12ea 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/r3labs/diff" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -723,6 +724,17 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } + // connection pool + if !reflect.DeepEqual(oldSpec.Spec.ConnectionPool, + newSpec.Spec.ConnectionPool) { + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPool(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + updateFailed = true + } + } + return nil } @@ -852,13 +864,13 @@ func (c *Cluster) initSystemUsers() { if c.needConnectionPool() { username := c.Spec.ConnectionPool.User - if username == nil { - username = &c.OpConfig.ConnectionPool.User + if username == "" { + username = c.OpConfig.ConnectionPool.User } c.systemUsers[constants.ConnectionPoolUserKeyName] = spec.PgUser{ Origin: spec.RoleConnectionPool, - Name: *username, + Name: username, Password: util.RandomPassword(constants.PasswordLength), } } @@ -1188,3 +1200,29 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } + +// Test if two connection pool configuration needs to be synced. For simplicity +// compare not the actual K8S objects, but the configuration itself and request +// sync if there is any difference. +func (c *Cluster) needSyncConnPoolDeployments(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { + reasons = []string{} + sync = false + + changelog, err := diff.Diff(oldSpec, newSpec) + if err != nil { + c.logger.Infof("Cannot get diff, do not do anything, %+v", err) + return false, reasons + } else { + if len(changelog) > 0 { + sync = true + } + + for _, change := range changelog { + msg := fmt.Sprintf("%s %+v from %s to %s", + change.Type, change.Path, change.From, change.To) + reasons = append(reasons, msg) + } + } + + return sync, reasons +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 1b74bd6b6..0c1e07a11 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -300,8 +300,26 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { params, err) } - if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { - return fmt.Errorf("could not execute sql statement %s: %v", + // golang sql will do retries couple of times if pq driver reports + // connections issues (driver.ErrBadConn), but since our query is + // idempotent, we can retry in a view of other errors (e.g. due to + // failover a db is temporary in a read-only mode or so) to make sure + // it was applied. + execErr := retryutil.Retry( + constants.PostgresConnectTimeout, + constants.PostgresConnectRetryTimeout, + func() (bool, error) { + if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + msg := fmt.Errorf("could not execute sql statement %s: %v", + stmtBytes.String(), err) + return false, msg + } + + return true, nil + }) + + if execErr != nil { + return fmt.Errorf("could not execute after retries %s: %v", stmtBytes.String(), err) } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 8b7860886..3121691d9 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1734,100 +1734,99 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( *v1.PodTemplateSpec, error) { - podTemplate := spec.ConnectionPool.PodTemplate + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + spec.ConnectionPool.Resources, + c.makeDefaultConnPoolResources()) - if podTemplate == nil { - gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) - resources, err := generateResourceRequirements( - spec.ConnectionPool.Resources, - c.makeDefaultConnPoolResources()) + effectiveMode := util.Coalesce( + spec.ConnectionPool.Mode, + c.OpConfig.ConnectionPool.Mode) - effectiveMode := spec.ConnectionPool.Mode - if effectiveMode == nil { - effectiveMode = &c.OpConfig.ConnectionPool.Mode - } + effectiveDockerImage := util.Coalesce( + spec.ConnectionPool.DockerImage, + c.OpConfig.ConnectionPool.Image) - if err != nil { - return nil, fmt.Errorf("could not generate resource requirements: %v", err) - } + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } - secretSelector := func(key string) *v1.SecretKeySelector { - return &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: c.credentialSecretName(c.OpConfig.SuperUsername), - }, - Key: key, - } + secretSelector := func(key string) *v1.SecretKeySelector { + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(c.OpConfig.SuperUsername), + }, + Key: key, } + } - envVars := []v1.EnvVar{ - { - Name: "PGHOST", - Value: c.serviceAddress(Master), - }, - { - Name: "PGPORT", - Value: c.servicePort(Master), - }, - { - Name: "PGUSER", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("username"), - }, - }, - // the convention is to use the same schema name as - // connection pool username - { - Name: "PGSCHEMA", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("username"), - }, - }, - { - Name: "PGPASSWORD", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("password"), - }, + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(Master), + }, + { + Name: "PGPORT", + Value: c.servicePort(Master), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), }, - { - Name: "CONNECTION_POOL_MODE", - Value: *effectiveMode, + }, + // the convention is to use the same schema name as + // connection pool username + { + Name: "PGSCHEMA", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), }, - { - Name: "CONNECTION_POOL_PORT", - Value: fmt.Sprint(pgPort), + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), }, - } + }, + { + Name: "CONNECTION_POOL_MODE", + Value: effectiveMode, + }, + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + } - poolerContainer := v1.Container{ - Name: connectionPoolContainer, - Image: c.OpConfig.ConnectionPool.Image, - ImagePullPolicy: v1.PullIfNotPresent, - Resources: *resources, - Ports: []v1.ContainerPort{ - { - ContainerPort: pgPort, - Protocol: v1.ProtocolTCP, - }, + poolerContainer := v1.Container{ + Name: connectionPoolContainer, + Image: effectiveDockerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, }, - Env: envVars, - } + }, + Env: envVars, + } - podTemplate = &v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: c.connPoolLabelsSelector().MatchLabels, - Namespace: c.Namespace, - Annotations: c.generatePodAnnotations(spec), - }, - Spec: v1.PodSpec{ - ServiceAccountName: c.OpConfig.PodServiceAccountName, - TerminationGracePeriodSeconds: &gracePeriod, - Containers: []v1.Container{poolerContainer}, - // TODO: add tolerations to scheduler pooler on the same node - // as database - //Tolerations: *tolerationsSpec, - }, - } + podTemplate := &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connPoolLabelsSelector().MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, } return podTemplate, nil diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 012df4072..c26e04b96 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -613,21 +613,6 @@ func TestConnPoolPodSpec(t *testing.T) { cluster: cluster, check: testEnvs, }, - { - subTest: "custom pod template", - spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{ - PodTemplate: &v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-template", - }, - }, - }, - }, - expected: nil, - cluster: cluster, - check: testCustomPodTemplate, - }, } for _, tt := range tests { podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 4f9d72e19..e44d50800 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -101,9 +101,17 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolR var msg string c.setProcessName("creating connection pool") - err := lookup( - c.OpConfig.ConnectionPool.Schema, - c.OpConfig.ConnectionPool.User) + schema := c.Spec.ConnectionPool.Schema + if schema == "" { + schema = c.OpConfig.ConnectionPool.Schema + } + + user := c.Spec.ConnectionPool.User + if user == "" { + user = c.OpConfig.ConnectionPool.User + } + + err := lookup(schema, user) if err != nil { msg = "could not prepare database for connection pool: %v" @@ -116,6 +124,9 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolR return nil, fmt.Errorf(msg, err) } + // client-go does retry 10 times (with NoBackoff by default) when the API + // believe a request can be retried and returns Retry-After header. This + // should be good enough to not think about it here. deployment, err := c.KubeClient. Deployments(deploymentSpec.Namespace). Create(deploymentSpec) @@ -154,14 +165,22 @@ func (c *Cluster) deleteConnectionPool() (err error) { return nil } + // Clean up the deployment object. If deployment resource we've remembered + // is somehow empty, try to delete based on what would we generate + deploymentName := c.connPoolName() + deployment := c.ConnectionPool.Deployment + + if deployment != nil { + deploymentName = deployment.Name + } + // set delete propagation policy to foreground, so that replica set will be // also deleted. policy := metav1.DeletePropagationForeground options := metav1.DeleteOptions{PropagationPolicy: &policy} - deployment := c.ConnectionPool.Deployment err = c.KubeClient. - Deployments(deployment.Namespace). - Delete(deployment.Name, &options) + Deployments(c.Namespace). + Delete(deploymentName, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool deployment was already deleted") @@ -172,12 +191,19 @@ func (c *Cluster) deleteConnectionPool() (err error) { c.logger.Infof("Connection pool deployment %q has been deleted", util.NameFromMeta(deployment.ObjectMeta)) + // Repeat the same for the service object + service := c.ConnectionPool.Service + serviceName := c.connPoolName() + + if service != nil { + serviceName = service.Name + } + // set delete propagation policy to foreground, so that all the dependant // will be deleted. - service := c.ConnectionPool.Service err = c.KubeClient. - Services(service.Namespace). - Delete(service.Name, &options) + Services(c.Namespace). + Delete(serviceName, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool service was already deleted") @@ -823,3 +849,34 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } + +// Perform actual patching of a connection pool deployment, assuming that all +// the check were already done before. +func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + c.setProcessName("updating connection pool") + if c.ConnectionPool == nil || c.ConnectionPool.Deployment == nil { + return nil, fmt.Errorf("there is no connection pool in the cluster") + } + + patchData, err := specPatch(newDeployment.Spec) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment: %v", err) + } + + // An update probably requires RetryOnConflict, but since only one operator + // worker at one time will try to update it changes of conflicts are + // minimal. + deployment, err := c.KubeClient. + Deployments(c.ConnectionPool.Deployment.Namespace). + Patch( + c.ConnectionPool.Deployment.Name, + types.MergePatchType, + patchData, "") + if err != nil { + return nil, fmt.Errorf("could not patch deployment: %v", err) + } + + c.ConnectionPool.Deployment = deployment + + return deployment, nil +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 5ee827d4b..b59bf5533 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -2,6 +2,7 @@ package cluster import ( "fmt" + "reflect" batchv1beta1 "k8s.io/api/batch/v1beta1" v1 "k8s.io/api/core/v1" @@ -23,6 +24,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() + oldSpec := c.Postgresql c.setSpec(newSpec) defer func() { @@ -108,6 +110,20 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + // connection pool + oldPool := oldSpec.Spec.ConnectionPool + newPool := newSpec.Spec.ConnectionPool + if c.needConnectionPool() && + (c.ConnectionPool == nil || !reflect.DeepEqual(oldPool, newPool)) { + + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPool(&oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } + return err } @@ -594,3 +610,98 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } + +// Synchronize connection pool resources. Effectively we're interested only in +// synchronizing the corresponding deployment, but in case of deployment or +// service is missing, create it. After checking, also remember an object for +// the future references. +func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error { + if c.ConnectionPool == nil { + c.logger.Warning("Connection pool resources are empty") + c.ConnectionPool = &ConnectionPoolResources{} + } + + deployment, err := c.KubeClient. + Deployments(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Deployment %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + deploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + } else if err != nil { + return fmt.Errorf("could not get connection pool deployment to sync: %v", err) + } else { + c.ConnectionPool.Deployment = deployment + + // actual synchronization + oldConnPool := oldSpec.Spec.ConnectionPool + newConnPool := newSpec.Spec.ConnectionPool + sync, reason := c.needSyncConnPoolDeployments(oldConnPool, newConnPool) + if sync { + c.logger.Infof("Update connection pool deployment %s, reason: %s", + c.connPoolName(), reason) + + newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg := "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + oldDeploymentSpec := c.ConnectionPool.Deployment + + deployment, err := c.updateConnPoolDeployment( + oldDeploymentSpec, + newDeploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + return nil + } + } + + service, err := c.KubeClient. + Services(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Service %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + serviceSpec := c.generateConnPoolService(&newSpec.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Service = service + } else if err != nil { + return fmt.Errorf("could not get connection pool service to sync: %v", err) + } else { + // Service updates are not supported and probably not that useful anyway + c.ConnectionPool.Service = service + } + + return nil +} diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go new file mode 100644 index 000000000..c6928a64e --- /dev/null +++ b/pkg/cluster/sync_test.go @@ -0,0 +1,125 @@ +package cluster + +import ( + "fmt" + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func int32ToPointer(value int32) *int32 { + return &value +} + +func deploymentUpdated(cluster *Cluster, err error) error { + if cluster.ConnectionPool.Deployment.Spec.Replicas == nil || + *cluster.ConnectionPool.Deployment.Spec.Replicas != 2 { + return fmt.Errorf("Wrong nubmer of instances") + } + + return nil +} + +func objectsAreSaved(cluster *Cluster, err error) error { + if cluster.ConnectionPool == nil { + return fmt.Errorf("Connection pool resources are empty") + } + + if cluster.ConnectionPool.Deployment == nil { + return fmt.Errorf("Deployment was not saved") + } + + if cluster.ConnectionPool.Service == nil { + return fmt.Errorf("Service was not saved") + } + + return nil +} + +func TestConnPoolSynchronization(t *testing.T) { + testName := "Test connection pool synchronization" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + clusterMissingObjects := *cluster + clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() + + clusterMock := *cluster + clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() + + tests := []struct { + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + check func(cluster *Cluster, err error) error + }{ + { + subTest: "create if doesn't exist", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "update deployment", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(1), + }, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(2), + }, + }, + }, + cluster: &clusterMock, + check: deploymentUpdated, + }, + } + for _, tt := range tests { + err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec) + + if err := tt.check(tt.cluster, err); err != nil { + t.Errorf("%s [%s]: Could not synchronize, %+v", + testName, tt.subTest, err) + } + } +} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index d5f9c744f..d2dd11586 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -497,5 +497,5 @@ func (c *Cluster) patroniUsesKubernetes() bool { } func (c *Cluster) needConnectionPool() bool { - return c.Spec.ConnectionPool != nil + return c.Spec.ConnectionPool != nil || c.Spec.EnableConnectionPool == true } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index f5d280363..1748fbd1f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -6,7 +6,9 @@ import ( "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -142,17 +144,52 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit - // connection pool + // Connection pool. Looks like we can't use defaulting in CRD before 1.17, + // so ensure default values here. result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances - result.ConnectionPool.Schema = fromCRD.ConnectionPool.Schema - result.ConnectionPool.User = fromCRD.ConnectionPool.User - result.ConnectionPool.Type = fromCRD.ConnectionPool.Type - result.ConnectionPool.Image = fromCRD.ConnectionPool.Image - result.ConnectionPool.Mode = fromCRD.ConnectionPool.Mode - result.ConnectionPool.ConnPoolDefaultCPURequest = fromCRD.ConnectionPool.DefaultCPURequest - result.ConnectionPool.ConnPoolDefaultMemoryRequest = fromCRD.ConnectionPool.DefaultMemoryRequest - result.ConnectionPool.ConnPoolDefaultCPULimit = fromCRD.ConnectionPool.DefaultCPULimit - result.ConnectionPool.ConnPoolDefaultMemoryLimit = fromCRD.ConnectionPool.DefaultMemoryLimit + if result.ConnectionPool.NumberOfInstances == nil || + *result.ConnectionPool.NumberOfInstances < 1 { + var value int32 + + value = 1 + result.ConnectionPool.NumberOfInstances = &value + } + + result.ConnectionPool.Schema = util.Coalesce( + fromCRD.ConnectionPool.Schema, + constants.ConnectionPoolSchemaName) + + result.ConnectionPool.User = util.Coalesce( + fromCRD.ConnectionPool.User, + constants.ConnectionPoolUserName) + + result.ConnectionPool.Type = util.Coalesce( + fromCRD.ConnectionPool.Type, + constants.ConnectionPoolDefaultType) + + result.ConnectionPool.Image = util.Coalesce( + fromCRD.ConnectionPool.Image, + "pgbouncer:0.0.1") + + result.ConnectionPool.Mode = util.Coalesce( + fromCRD.ConnectionPool.Mode, + constants.ConnectionPoolDefaultMode) + + result.ConnectionPool.ConnPoolDefaultCPURequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPURequest, + constants.ConnectionPoolDefaultCpuRequest) + + result.ConnectionPool.ConnPoolDefaultMemoryRequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryRequest, + constants.ConnectionPoolDefaultMemoryRequest) + + result.ConnectionPool.ConnPoolDefaultCPULimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPULimit, + constants.ConnectionPoolDefaultCpuLimit) + + result.ConnectionPool.ConnPoolDefaultMemoryLimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryLimit, + constants.ConnectionPoolDefaultMemoryLimit) return result } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go new file mode 100644 index 000000000..b25a12a6c --- /dev/null +++ b/pkg/util/constants/pooler.go @@ -0,0 +1,13 @@ +package constants + +// Connection pool specific constants +const ( + ConnectionPoolUserName = "pooler" + ConnectionPoolSchemaName = "pooler" + ConnectionPoolDefaultType = "pgbouncer" + ConnectionPoolDefaultMode = "transition" + ConnectionPoolDefaultCpuRequest = "100m" + ConnectionPoolDefaultCpuLimit = "100m" + ConnectionPoolDefaultMemoryRequest = "100M" + ConnectionPoolDefaultMemoryLimit = "100M" +) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 77e46476b..44a293025 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -16,6 +16,7 @@ import ( apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -28,6 +29,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func int32ToPointer(value int32) *int32 { + return &value +} + // KubernetesClient describes getters for Kubernetes objects type KubernetesClient struct { corev1.SecretsGetter @@ -62,16 +67,30 @@ type mockDeployment struct { appsv1.DeploymentInterface } +type mockDeploymentNotExist struct { + appsv1.DeploymentInterface +} + type MockDeploymentGetter struct { } +type MockDeploymentNotExistGetter struct { +} + type mockService struct { corev1.ServiceInterface } +type mockServiceNotExist struct { + corev1.ServiceInterface +} + type MockServiceGetter struct { } +type MockServiceNotExistGetter struct { +} + type mockConfigMap struct { corev1.ConfigMapInterface } @@ -245,6 +264,10 @@ func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.Deploymen return &mockDeployment{} } +func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeploymentNotExist{} +} + func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -257,10 +280,49 @@ func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) erro return nil } +func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + Spec: apiappsv1.DeploymentSpec{ + Replicas: int32ToPointer(2), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + +func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface { return &mockService{} } +func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.ServiceInterface { + return &mockServiceNotExist{} +} + func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -273,6 +335,30 @@ func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { return nil } +func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + // NewMockKubernetesClient for other tests func NewMockKubernetesClient() KubernetesClient { return KubernetesClient{ @@ -282,3 +368,10 @@ func NewMockKubernetesClient() KubernetesClient { ServicesGetter: &MockServiceGetter{}, } } + +func ClientMissingObjects() KubernetesClient { + return KubernetesClient{ + DeploymentsGetter: &MockDeploymentNotExistGetter{}, + ServicesGetter: &MockServiceNotExistGetter{}, + } +} From 55873f06be6df7d18260a60710fa1f77e6a867c9 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 11:04:11 +0100 Subject: [PATCH 06/61] Add test for both ways to enable connection pool --- pkg/cluster/resources_test.go | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index a3754c564..7c52addad 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -63,3 +63,41 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { t.Errorf("%s: Cannot delete connection pool, %s", testName, err) } } + +func TestNeedConnPool(t *testing.T) { + testName := "Test how connection pool can be enabled" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with full definition", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: true, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag", + testName) + } +} From b66b1633a145d52c48d737083b3dfda15ad0acf8 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 11:56:05 +0100 Subject: [PATCH 07/61] add validation for postgresql CRD --- .../crds/operatorconfigurations.yaml | 37 ++++++++++++++++ .../postgres-operator/crds/postgresqls.yaml | 44 +++++++++++++++++++ manifests/postgresql.crd.yaml | 44 +++++++++++++++++++ 3 files changed, 125 insertions(+) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 9725c2708..c99d4a811 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -318,6 +318,43 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" status: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index b4b676236..8b7de363c 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -106,6 +106,50 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + mode: + type: string + numberOfInstances: + type: integer + minimum: 1 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 276bc94b8..7d4bb228b 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -70,6 +70,50 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + mode: + type: string + numberOfInstances: + type: integer + minimum: 1 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: From f2c990512311503cb84fbbe72c8dd6eba4850351 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 12:54:32 +0100 Subject: [PATCH 08/61] reflect connectionPool validation in Go code and publish in manifests and chart --- .../crds/operatorconfigurations.yaml | 38 +++---- .../templates/configmap.yaml | 1 + .../templates/operatorconfiguration.yaml | 2 + charts/postgres-operator/values-crd.yaml | 11 ++ charts/postgres-operator/values.yaml | 12 ++ manifests/configmap.yaml | 9 ++ manifests/operatorconfiguration.crd.yaml | 38 +++---- ...gresql-operator-default-configuration.yaml | 10 ++ pkg/apis/acid.zalan.do/v1/crds.go | 104 ++++++++++++++++++ 9 files changed, 187 insertions(+), 38 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c99d4a811..a790125d1 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -321,24 +321,6 @@ spec: connection_pool: type: object properties: - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -354,7 +336,25 @@ spec: connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100m" + #default: "100Mi" + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" status: type: object additionalProperties: diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 95eeb9546..634339795 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -20,4 +20,5 @@ data: {{ toYaml .Values.configDebug | indent 2 }} {{ toYaml .Values.configLoggingRestApi | indent 2 }} {{ toYaml .Values.configTeamsApi | indent 2 }} +{{ toYaml .Values.configConnectionPool | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 6a301c1fb..55eb8fd4f 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -34,4 +34,6 @@ configuration: {{ toYaml .Values.configLoggingRestApi | indent 4 }} scalyr: {{ toYaml .Values.configScalyr | indent 4 }} + connection_pool: +{{ toYaml .Values.configConnectionPool | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 1f9b5e495..17b62226a 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -261,6 +261,17 @@ configScalyr: # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi +configConnectionPool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "1" + connection_pool_default_memory_limit: 100m + connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + connection_pool_instances_number: 1 + connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 1be5851d2..f11619c8a 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -237,6 +237,18 @@ configTeamsApi: # URL of the Teams API service # teams_api_url: http://fake-teams-api.default.svc.cluster.local +# configure connection pooler deployment created by the operator +configConnectionPool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "1" + connection_pool_default_memory_limit: 100m + connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + connection_pool_instances_number: 1 + connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d26c83edf..05f22a388 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,6 +11,15 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: version + # connection_pool_default_cpu_limit: "1" + # connection_pool_default_cpu_request: "1" + # connection_pool_default_memory_limit: 100m + # connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + # connection_pool_instances_number: 1 + # connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index c44955771..f4224244d 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -297,24 +297,6 @@ spec: connection_pool: type: object properties: - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -330,7 +312,25 @@ spec: connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100m" + #default: "100Mi" + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" status: type: object additionalProperties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index efd1a5396..037ae5e35 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -121,3 +121,13 @@ configuration: scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" + connection_pool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "1" + connection_pool_default_memory_limit: 100m + connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + connection_pool_instances_number: 1 + connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 4cfc9a9e6..f760d63e5 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -176,6 +176,65 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "connectionPool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "dockerImage": { + Type: "string", + }, + "mode": { + Type: "string", + }, + "numberOfInstances": { + Type: "integer", + Minimum: &min1, + }, + "resources": { + Type: "object", + Required: []string{"requests", "limits"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "limits": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + "requests": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + }, + }, + "schema": { + Type: "string", + }, + "user": { + Type: "string", + }, + }, + }, "databases": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ @@ -1037,6 +1096,51 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "connection_pool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "connection_pool_default_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_default_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_image": { + Type: "string", + }, + "connection_pool_instances_number": { + Type: "integer", + Minimum: &min1, + }, + "connection_pool_mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, + }, + "connection_pool_schema": { + Type: "string", + }, + "connection_pool_user": { + Type: "string", + }, + }, + }, }, }, "status": { From 6dad83325b81192b96637ad7fa41b1ceea124146 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 13:21:20 +0100 Subject: [PATCH 09/61] Cleanup configuration Add pool configuration into CRD & charts. Add preliminary documentation. Rename NumberOfInstances to Replicas like in Deployment. Mention couple of potential improvement points for connection pool specification. --- .../crds/operatorconfigurations.yaml | 38 ++++++------- .../postgres-operator/crds/postgresqls.yaml | 16 ++++++ charts/postgres-operator/values-crd.yaml | 24 +++++---- charts/postgres-operator/values.yaml | 22 +++++--- docs/reference/cluster_manifest.md | 27 ++++++++++ docs/reference/operator_parameters.md | 28 ++++++++++ docs/user.md | 53 +++++++++++++++++++ manifests/operatorconfiguration.crd.yaml | 36 ++++++------- manifests/postgresql.crd.yaml | 16 ++++++ .../v1/operator_configuration_type.go | 15 +++--- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 25 +++++---- pkg/cluster/k8sres.go | 4 +- pkg/cluster/sync_test.go | 4 +- pkg/controller/operator_config.go | 12 ++--- pkg/util/config/config.go | 3 +- 15 files changed, 237 insertions(+), 86 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index a790125d1..4c5ebdf66 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -321,6 +321,24 @@ spec: connection_pool: type: object properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -336,25 +354,7 @@ spec: connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" + #default: "100m" status: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 8b7de363c..aa4d40b1d 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -242,6 +242,22 @@ spec: type: string replicaLoadBalancer: # deprecated type: boolean + connectionPool: + type: object + properties: + schema: + type: string + user: + type: string + replicas: + type: integer + dockerImage: + type: string + mode: + type: string + enum: + - "session" + - "transaction" resources: type: object required: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 17b62226a..cc16a0979 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -262,15 +262,21 @@ configScalyr: scalyr_memory_request: 50Mi configConnectionPool: - connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "1" - connection_pool_default_memory_limit: 100m - connection_pool_default_memory_request: "100Mi" - # connection_pool_image: "" - connection_pool_instances_number: 1 - connection_pool_mode: "transaction" - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + # number of pooler instances + connection_pool_replicas: 1 + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # default pooling mode + connection_pool_mode: "transaction" + # default resources + connection_pool_default_cpu_request: "100m" + connection_pool_default_memory_request: "100M" + connection_pool_default_cpu_limit: "100m" + connection_pool_default_memory_limit: "100M" rbac: # Specifies whether RBAC resources should be created diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index f11619c8a..1f4cb6f70 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -239,15 +239,21 @@ configTeamsApi: # configure connection pooler deployment created by the operator configConnectionPool: - connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "1" - connection_pool_default_memory_limit: 100m - connection_pool_default_memory_request: "100Mi" - # connection_pool_image: "" - connection_pool_instances_number: 1 + # number of pooler instances + connection_pool_replicas: 1 + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # default pooling mode connection_pool_mode: "transaction" - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + # default resources + connection_pool_default_cpu_request: "100m" + connection_pool_default_memory_request: "100M" + connection_pool_default_cpu_limit: "100m" + connection_pool_default_memory_limit: "100M" rbac: # Specifies whether RBAC resources should be created diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 7b049b6fa..a49890d13 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -149,6 +149,11 @@ These parameters are grouped directly under the `spec` key in the manifest. [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Optional. Default is: "30 00 \* \* \*" +* enableConnectionPool + Tells the operator to create a connection pool with a database. If this + field is true, a connection pool deployment will be created even if + `connectionPool` section is empty. + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is @@ -359,3 +364,25 @@ CPU and memory limits for the sidecar container. * **memory** memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. + +## Connection pool + +Parameters are grouped under the `connectionPool` top-level key and specify +configuration for connection pool. If this section is not empty, a connection +pool will be created for a database even if `enableConnectionPool` is not +present. + +* **replicas** + How many instances of connection pool to create. + +* **mode** + In which mode to run connection pool, transaction or section. + +* **schema** + Schema to create for credentials lookup function. + +* **user** + User to create for connection pool to be able to connect to a database. + +* **resources** + Resource configuration for connection pool deployment. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index e3893ea31..52d4e66c1 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -592,3 +592,31 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the * **scalyr_memory_limit** Memory limit value for the Scalyr sidecar. The default is `500Mi`. + +## Connection pool configuration + +Parameters are grouped under the `connection_pool` top-level key and specify +default configuration for connection pool, if a postgres manifest requests it +but do not specify some of the parameters. All of them are optional with the +operator being able to provide some reasonable defaults. + +* **connection_pool_replicas** + How many instances of connection pool to create. + +* **connection_pool_schema** + Schema to create for credentials lookup function. + +* **connection_pool_user** + User to create for connection pool to be able to connect to a database. + +* **connection_pool_image** + Docker image to use for connection pool deployment. + +* **connection_pool_mode** + Default pool mode, sesssion or transaction. + +* **connection_pool_default_cpu_request** + **connection_pool_default_memory_reques** + **connection_pool_default_cpu_limit** + **connection_pool_default_memory_limit** + Default resource configuration for connection pool deployment. diff --git a/docs/user.md b/docs/user.md index f81e11ede..47be76773 100644 --- a/docs/user.md +++ b/docs/user.md @@ -454,3 +454,56 @@ monitoring is outside the scope of operator responsibilities. See [configuration reference](reference/cluster_manifest.md) and [administrator documentation](administrator.md) for details on how backups are executed. + +## Connection pool + +The operator can create a database side connection pool for those applications, +where an application side pool is not feasible, but a number of connections is +high. To create a connection pool together with a database, modify the +manifest: + +```yaml +spec: + enableConnectionPool: true +``` + +This will tell the operator to create a connection pool with default +configuration, though which one can access the master via a separate service +`{cluster-name}-pooler`. In most of the cases provided default configuration +should be good enough. + +To configure a new connection pool, specify: + +``` +spec: + connectionPool: + # how many instances of connection pool to create + replicas: 1 + + # in which mode to run, session or transaction + mode: "transaction" + + # schema, which operator will create to install credentials lookup + # function + schema: "pooler" + + # user, which operator will create for connection pool + user: "pooler" + + # resources for each instance + resources: + requests: + cpu: "100m" + memory: "100M" + limits: + cpu: "100m" + memory: "100M" +``` + +By default `pgbouncer` is used to create a connection pool. To find out about +pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it +should be general approach between different implementation). + +Note, that using `pgbouncer` means meaningful resource CPU limit should be less +than 1 core (there is a way to utilize more than one, but in K8S it's easier +just to spin up more instances). diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index f4224244d..aa64c6f6d 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -297,6 +297,24 @@ spec: connection_pool: type: object properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -313,24 +331,6 @@ spec: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" status: type: object additionalProperties: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 7d4bb228b..ff9366421 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -206,6 +206,22 @@ spec: type: string replicaLoadBalancer: # deprecated type: boolean + connectionPool: + type: object + properties: + schema: + type: string + user: + type: string + replicas: + type: integer + dockerImage: + type: string + mode: + type: string + enum: + - "session" + - "transaction" resources: type: object required: diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 58f171843..dd0822c6a 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -65,12 +65,12 @@ type KubernetesMetaConfiguration struct { // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname - PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + PodManagementPolicy string `json:"pod_management_policy,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources @@ -154,10 +154,9 @@ type ScalyrConfiguration struct { // Defines default configuration for connection pool type ConnectionPoolConfiguration struct { - NumberOfInstances *int32 `json:"connection_pool_instances_number,omitempty"` + Replicas *int32 `json:"connection_pool_replicas,omitempty"` Schema string `json:"connection_pool_schema,omitempty"` User string `json:"connection_pool_user,omitempty"` - Type string `json:"connection_pool_type,omitempty"` Image string `json:"connection_pool_image,omitempty"` Mode string `json:"connection_pool_mode,omitempty"` DefaultCPURequest string `name:"connection_pool_default_cpu_request,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index c56d70626..e7965f893 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,7 +27,7 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - EnableConnectionPool bool `json:"enable_connection_pool,omitempty"` + EnableConnectionPool bool `json:"enableConnectionPool,omitempty"` ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` TeamID string `json:"teamId"` @@ -159,16 +159,21 @@ type PostgresStatus struct { } // Options for connection pooler +// +// TODO: prepared snippets of configuration, one can choose via type, e.g. +// pgbouncer-large (with higher resources) or odyssey-small (with smaller +// resources) +// Type string `json:"type,omitempty"` +// +// TODO: figure out what other important parameters of the connection pool it +// makes sense to expose. E.g. pool size (min/max boundaries), max client +// connections etc. type ConnectionPool struct { - NumberOfInstances *int32 `json:"instancesNumber,omitempty"` - Schema string `json:"schema,omitempty"` - User string `json:"user,omitempty"` - Mode string `json:"mode,omitempty"` - DockerImage string `json:"dockerImage,omitempty"` - // TODO: prepared snippets of configuration, one can choose via type, e.g. - // pgbouncer-large (with higher resources) or odyssey-small (with smaller - // resources) - // Type string `json:"type,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 3121691d9..b9f8e1992 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1861,9 +1861,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { podTemplate, err := c.generateConnPoolPodTemplate(spec) - numberOfInstances := spec.ConnectionPool.NumberOfInstances + numberOfInstances := spec.ConnectionPool.Replicas if numberOfInstances == nil { - numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances + numberOfInstances = c.OpConfig.ConnectionPool.Replicas } if err != nil { diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index c6928a64e..f5887dede 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -99,14 +99,14 @@ func TestConnPoolSynchronization(t *testing.T) { oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{ - NumberOfInstances: int32ToPointer(1), + Replicas: int32ToPointer(1), }, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{ - NumberOfInstances: int32ToPointer(2), + Replicas: int32ToPointer(2), }, }, }, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 1748fbd1f..a4a32abba 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -146,13 +146,13 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // Connection pool. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. - result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances - if result.ConnectionPool.NumberOfInstances == nil || - *result.ConnectionPool.NumberOfInstances < 1 { + result.ConnectionPool.Replicas = fromCRD.ConnectionPool.Replicas + if result.ConnectionPool.Replicas == nil || + *result.ConnectionPool.Replicas < 1 { var value int32 value = 1 - result.ConnectionPool.NumberOfInstances = &value + result.ConnectionPool.Replicas = &value } result.ConnectionPool.Schema = util.Coalesce( @@ -163,10 +163,6 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur fromCRD.ConnectionPool.User, constants.ConnectionPoolUserName) - result.ConnectionPool.Type = util.Coalesce( - fromCRD.ConnectionPool.Type, - constants.ConnectionPoolDefaultType) - result.ConnectionPool.Image = util.Coalesce( fromCRD.ConnectionPool.Image, "pgbouncer:0.0.1") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2baf99931..b04206d53 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,10 +85,9 @@ type LogicalBackup struct { // Operator options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` + Replicas *int32 `name:"connection_pool_replicas" default:"1"` Schema string `name:"connection_pool_schema" default:"pooler"` User string `name:"connection_pool_user" default:"pooler"` - Type string `name:"connection_pool_type" default:"pgbouncer"` Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` Mode string `name:"connection_pool_mode" default:"session"` ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"100m"` From be438b77e6807f903e816f3bb08123f27fcd0a15 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Tue, 14 Jan 2020 15:36:30 +0100 Subject: [PATCH 10/61] WIP Connection pooler support Add an initial support for a connection pooler. The idea is to make it generic enough to be able to switch a corresponding docker image to change from pgbouncer to e.g. odyssey. Operator needs to create a deployment with pooler and a service for it to access. --- docker/DebugDockerfile | 13 +- manifests/minimal-postgres-manifest.yaml | 2 + manifests/operator-service-account-rbac.yaml | 1 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 12 ++ pkg/apis/acid.zalan.do/v1/register.go | 5 +- pkg/cluster/cluster.go | 55 ++++- pkg/cluster/database.go | 67 ++++++ pkg/cluster/k8sres.go | 216 ++++++++++++++++++- pkg/cluster/resources.go | 96 +++++++++ pkg/cluster/sync.go | 6 + pkg/cluster/types.go | 2 + pkg/cluster/util.go | 18 +- pkg/spec/types.go | 4 +- pkg/util/config/config.go | 11 + pkg/util/constants/roles.go | 23 +- pkg/util/k8sutil/k8sutil.go | 2 + 16 files changed, 513 insertions(+), 20 deletions(-) diff --git a/docker/DebugDockerfile b/docker/DebugDockerfile index 76dadf6df..0c11fe3b4 100644 --- a/docker/DebugDockerfile +++ b/docker/DebugDockerfile @@ -3,8 +3,17 @@ MAINTAINER Team ACID @ Zalando # We need root certificates to deal with teams api over https RUN apk --no-cache add ca-certificates go git musl-dev -RUN go get github.com/derekparker/delve/cmd/dlv COPY build/* / -CMD ["/root/go/bin/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] +RUN addgroup -g 1000 pgo +RUN adduser -D -u 1000 -G pgo -g 'Postgres Operator' pgo + +RUN go get github.com/derekparker/delve/cmd/dlv +RUN cp /root/go/bin/dlv /dlv +RUN chown -R pgo:pgo /dlv + +USER pgo:pgo +RUN ls -l / + +CMD ["/dlv", "--listen=:7777", "--headless=true", "--api-version=2", "exec", "/postgres-operator"] diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 75dfdf07f..ff7785cec 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -17,3 +17,5 @@ spec: foo: zalando # dbname: owner postgresql: version: "11" + connectionPool: + type: "pgbouncer" diff --git a/manifests/operator-service-account-rbac.yaml b/manifests/operator-service-account-rbac.yaml index a37abe476..d7cd6fd74 100644 --- a/manifests/operator-service-account-rbac.yaml +++ b/manifests/operator-service-account-rbac.yaml @@ -119,6 +119,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 07b42d4d4..c50d9c902 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,6 +27,8 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` + ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -154,3 +156,13 @@ type UserFlags []string type PostgresStatus struct { PostgresClusterStatus string `json:"PostgresClusterStatus"` } + +// Options for connection pooler +type ConnectionPool struct { + NumberOfInstances *int32 `json:"instancesNumber,omitempty"` + Schema *string `json:"schema,omitempty"` + User *string `json:"user,omitempty"` + Type *string `json:"type,omitempty"` + Mode *string `json:"mode,omitempty"` + PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` +} diff --git a/pkg/apis/acid.zalan.do/v1/register.go b/pkg/apis/acid.zalan.do/v1/register.go index 1c30e35fb..34def209d 100644 --- a/pkg/apis/acid.zalan.do/v1/register.go +++ b/pkg/apis/acid.zalan.do/v1/register.go @@ -10,7 +10,8 @@ import ( // APIVersion of the `postgresql` and `operator` CRDs const ( - APIVersion = "v1" + APIVersion = "v1" + PostgresqlKind = "postgresql" ) var ( @@ -42,7 +43,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { // AddKnownType assumes derives the type kind from the type name, which is always uppercase. // For our CRDs we use lowercase names historically, therefore we have to supply the name separately. // TODO: User uppercase CRDResourceKind of our types in the next major API version - scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresql"), &Postgresql{}) + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind(PostgresqlKind), &Postgresql{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresqlList"), &PostgresqlList{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("OperatorConfiguration"), &OperatorConfiguration{}) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index c560c4cdf..1681e7d2e 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -48,11 +48,17 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding } +type ConnectionPoolResources struct { + Deployment *appsv1.Deployment + Service *v1.Service +} + type kubeResources struct { Services map[PostgresRole]*v1.Service Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet + ConnectionPool *ConnectionPoolResources PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately @@ -184,7 +190,8 @@ func (c *Cluster) isNewCluster() bool { func (c *Cluster) initUsers() error { c.setProcessName("initializing users") - // clear our the previous state of the cluster users (in case we are running a sync). + // clear our the previous state of the cluster users (in case we are + // running a sync). c.systemUsers = map[string]spec.PgUser{} c.pgUsers = map[string]spec.PgUser{} @@ -292,8 +299,10 @@ func (c *Cluster) Create() error { } c.logger.Infof("pods are ready") - // create database objects unless we are running without pods or disabled that feature explicitly + // create database objects unless we are running without pods or disabled + // that feature explicitly if !(c.databaseAccessDisabled() || c.getNumberOfInstances(&c.Spec) <= 0 || c.Spec.StandbyCluster != nil) { + c.logger.Infof("Create roles") if err = c.createRoles(); err != nil { return fmt.Errorf("could not create users: %v", err) } @@ -316,6 +325,26 @@ func (c *Cluster) Create() error { c.logger.Errorf("could not list resources: %v", err) } + // Create connection pool deployment and services if necessary. Since we + // need to peform some operations with the database itself (e.g. install + // lookup function), do it as the last step, when everything is available. + // + // Do not consider connection pool as a strict requirement, and if + // something fails, report warning + if c.needConnectionPool() { + if c.ConnectionPool != nil { + c.logger.Warning("Connection pool already exists in the cluster") + return nil + } + connPool, err := c.createConnectionPool() + if err != nil { + c.logger.Warningf("could not create connection pool: %v", err) + return nil + } + c.logger.Infof("connection pool %q has been successfully created", + util.NameFromMeta(connPool.Deployment.ObjectMeta)) + } + return nil } @@ -745,6 +774,12 @@ func (c *Cluster) Delete() { c.logger.Warningf("could not remove leftover patroni objects; %v", err) } + // Delete connection pool objects anyway, even if it's not mentioned in the + // manifest, just to not keep orphaned components in case if something went + // wrong + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } } //NeedsRepair returns true if the cluster should be included in the repair scan (based on its in-memory status). @@ -811,6 +846,22 @@ func (c *Cluster) initSystemUsers() { Name: c.OpConfig.ReplicationUsername, Password: util.RandomPassword(constants.PasswordLength), } + + // Connection pool user is an exception, if requested it's going to be + // created by operator as a normal pgUser + if c.needConnectionPool() { + + username := c.Spec.ConnectionPool.User + if username == nil { + username = &c.OpConfig.ConnectionPool.User + } + + c.systemUsers[constants.ConnectionPoolUserKeyName] = spec.PgUser{ + Origin: spec.RoleConnectionPool, + Name: *username, + Password: util.RandomPassword(constants.PasswordLength), + } + } } func (c *Cluster) initRobotUsers() error { diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 07ea011a6..1b74bd6b6 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -1,10 +1,12 @@ package cluster import ( + "bytes" "database/sql" "fmt" "net" "strings" + "text/template" "time" "github.com/lib/pq" @@ -28,6 +30,25 @@ const ( getDatabasesSQL = `SELECT datname, pg_get_userbyid(datdba) AS owner FROM pg_database;` createDatabaseSQL = `CREATE DATABASE "%s" OWNER "%s";` alterDatabaseOwnerSQL = `ALTER DATABASE "%s" OWNER TO "%s";` + connectionPoolLookup = ` + CREATE SCHEMA IF NOT EXISTS {{.pool_schema}}; + + CREATE OR REPLACE FUNCTION {{.pool_schema}}.user_lookup( + in i_username text, out uname text, out phash text) + RETURNS record AS $$ + BEGIN + SELECT usename, passwd FROM pg_catalog.pg_shadow + WHERE usename = i_username INTO uname, phash; + RETURN; + END; + $$ LANGUAGE plpgsql SECURITY DEFINER; + + REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) + FROM public, {{.pool_schema}}; + GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) + TO {{.pool_user}}; + GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; + ` ) func (c *Cluster) pgConnectionString() string { @@ -243,3 +264,49 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin return result } + +// Creates a connection pool credentials lookup function in every database to +// perform remote authentification. +func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { + var stmtBytes bytes.Buffer + + if err := c.initDbConn(); err != nil { + return fmt.Errorf("could not init database connection") + } + defer func() { + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } + }() + + currentDatabases, err := c.getDatabases() + if err != nil { + msg := "could not get databases to install pool lookup function: %v" + return fmt.Errorf(msg, err) + } + + templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) + + for dbname, _ := range currentDatabases { + c.logger.Infof("Install pool lookup function into %s", dbname) + + params := TemplateParams{ + "pool_schema": poolSchema, + "pool_user": poolUser, + } + + if err := templater.Execute(&stmtBytes, params); err != nil { + return fmt.Errorf("could not prepare sql statement %+v: %v", + params, err) + } + + if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + return fmt.Errorf("could not execute sql statement %s: %v", + stmtBytes.String(), err) + } + + c.logger.Infof("Pool lookup function installed into %s", dbname) + } + + return nil +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index e6561e0f3..7d6b9be07 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -31,6 +31,8 @@ const ( patroniPGParametersParameterName = "parameters" patroniPGHBAConfParameterName = "pg_hba" localHost = "127.0.0.1/32" + connectionPoolContainer = "connection-pool" + pgPort = 5432 ) type pgUser struct { @@ -66,6 +68,10 @@ func (c *Cluster) statefulSetName() string { return c.Name } +func (c *Cluster) connPoolName() string { + return c.Name + "-pooler" +} + func (c *Cluster) endpointName(role PostgresRole) string { name := c.Name if role == Replica { @@ -84,6 +90,28 @@ func (c *Cluster) serviceName(role PostgresRole) string { return name } +func (c *Cluster) serviceAddress(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return service.ObjectMeta.Name + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + +func (c *Cluster) servicePort(role PostgresRole) string { + service, exist := c.Services[role] + + if exist { + return fmt.Sprint(service.Spec.Ports[0].Port) + } + + c.logger.Warningf("No service for role %s", role) + return "" +} + func (c *Cluster) podDisruptionBudgetName() string { return c.OpConfig.PDBNameFormat.Format("cluster", c.Name) } @@ -315,7 +343,11 @@ func tolerations(tolerationsSpec *[]v1.Toleration, podToleration map[string]stri return *tolerationsSpec } - if len(podToleration["key"]) > 0 || len(podToleration["operator"]) > 0 || len(podToleration["value"]) > 0 || len(podToleration["effect"]) > 0 { + if len(podToleration["key"]) > 0 || + len(podToleration["operator"]) > 0 || + len(podToleration["value"]) > 0 || + len(podToleration["effect"]) > 0 { + return []v1.Toleration{ { Key: podToleration["key"], @@ -1669,3 +1701,185 @@ func (c *Cluster) generateLogicalBackupPodEnvVars() []v1.EnvVar { func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } + +func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( + *v1.PodTemplateSpec, error) { + + podTemplate := spec.ConnectionPool.PodTemplate + + if podTemplate == nil { + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + c.Spec.Resources, + c.makeDefaultResources()) + + effectiveMode := spec.ConnectionPool.Mode + if effectiveMode == nil { + effectiveMode = &c.OpConfig.ConnectionPool.Mode + } + + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } + + secretSelector := func(key string) *v1.SecretKeySelector { + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(c.OpConfig.SuperUsername), + }, + Key: key, + } + } + + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(Master), + }, + { + Name: "PGPORT", + Value: c.servicePort(Master), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + // the convention is to use the same schema name as + // connection pool username + { + Name: "PGSCHEMA", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), + }, + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), + }, + }, + { + Name: "CONNECTION_POOL_MODE", + Value: *effectiveMode, + }, + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + } + + poolerContainer := v1.Container{ + Name: connectionPoolContainer, + Image: c.OpConfig.ConnectionPool.Image, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, + }, + }, + Env: envVars, + } + + podTemplate = &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connPoolLabelsSelector().MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, + } + } + + return podTemplate, nil +} + +func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( + *appsv1.Deployment, error) { + + podTemplate, err := c.generateConnPoolPodTemplate(spec) + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances + } + + if err != nil { + return nil, err + } + + deployment := &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: map[string]string{}, + // make Postgresql CRD object its owner, so that if CRD object is + // deleted, this object will be deleted even if something went + // wrong and operator didn't deleted it. + OwnerReferences: []metav1.OwnerReference{ + { + UID: c.Statefulset.ObjectMeta.UID, + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: c.Statefulset.ObjectMeta.Name, + }, + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: numberOfInstances, + Selector: c.connPoolLabelsSelector(), + Template: *podTemplate, + }, + } + + return deployment, nil +} + +func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { + serviceSpec := v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: c.connPoolName(), + Port: pgPort, + TargetPort: intstr.IntOrString{StrVal: c.servicePort(Master)}, + }, + }, + Type: v1.ServiceTypeClusterIP, + Selector: map[string]string{ + "connection-pool": c.connPoolName(), + }, + } + + service := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.connPoolName(), + Namespace: c.Namespace, + Labels: c.labelsSet(true), + Annotations: map[string]string{}, + // make Postgresql CRD object its owner, so that if CRD object is + // deleted, this object will be deleted even if something went + // wrong and operator didn't deleted it. + OwnerReferences: []metav1.OwnerReference{ + { + UID: c.Postgresql.ObjectMeta.UID, + APIVersion: acidv1.APIVersion, + Kind: acidv1.PostgresqlKind, + Name: c.Postgresql.ObjectMeta.Name, + }, + }, + }, + Spec: serviceSpec, + } + + return service +} diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index c94a7bb46..7baa96c02 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -90,6 +90,102 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { return statefulSet, nil } +// Prepare the database for connection pool to be used, i.e. install lookup +// function (do it first, because it should be fast and if it didn't succeed, +// it doesn't makes sense to create more K8S objects. At this moment we assume +// that necessary connection pool user exists. +// +// After that create all the objects for connection pool, namely a deployment +// with a chosen pooler and a service to expose it. +func (c *Cluster) createConnectionPool() (*ConnectionPoolResources, error) { + var msg string + c.setProcessName("creating connection pool") + + err := c.installLookupFunction( + c.OpConfig.ConnectionPool.Schema, + c.OpConfig.ConnectionPool.User) + + if err != nil { + msg = "could not prepare database for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + deploymentSpec, err := c.generateConnPoolDeployment(&c.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return nil, fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return nil, err + } + + serviceSpec := c.generateConnPoolService(&c.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return nil, err + } + + c.ConnectionPool = &ConnectionPoolResources{ + Deployment: deployment, + Service: service, + } + c.logger.Debugf("created new connection pool %q, uid: %q", + util.NameFromMeta(deployment.ObjectMeta), deployment.UID) + + return c.ConnectionPool, nil +} + +func (c *Cluster) deleteConnectionPool() (err error) { + c.setProcessName("deleting connection pool") + c.logger.Debugln("deleting connection pool") + + // Lack of connection pooler objects is not a fatal error, just log it if + // it was present before in the manifest + if c.needConnectionPool() && c.ConnectionPool == nil { + c.logger.Infof("No connection pool to delete") + return nil + } + + deployment := c.ConnectionPool.Deployment + err = c.KubeClient. + Deployments(deployment.Namespace). + Delete(deployment.Name, c.deleteOptions) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool deployment was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete deployment: %v", err) + } + + c.logger.Infof("Connection pool deployment %q has been deleted", + util.NameFromMeta(deployment.ObjectMeta)) + + service := c.ConnectionPool.Service + err = c.KubeClient. + Services(service.Namespace). + Delete(service.Name, c.deleteOptions) + + if !k8sutil.ResourceNotFound(err) { + c.logger.Debugf("Connection pool service was already deleted") + } else if err != nil { + return fmt.Errorf("could not delete service: %v", err) + } + + c.logger.Infof("Connection pool service %q has been deleted", + util.NameFromMeta(deployment.ObjectMeta)) + + c.ConnectionPool = nil + return nil +} + func getPodIndex(podName string) (int32, error) { parts := strings.Split(podName, "-") if len(parts) == 0 { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index fa4fc9ec1..5ee827d4b 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -456,6 +456,12 @@ func (c *Cluster) syncRoles() (err error) { for _, u := range c.pgUsers { userNames = append(userNames, u.Name) } + + // An exception from system users, connection pool user + connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] + userNames = append(userNames, connPoolUser.Name) + c.pgUsers[connPoolUser.Name] = connPoolUser + dbUsers, err = c.readPgUsersFromDatabase(userNames) if err != nil { return fmt.Errorf("error getting users from the database: %v", err) diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 138b7015c..286505621 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -69,3 +69,5 @@ type ClusterStatus struct { Spec acidv1.PostgresSpec Error error } + +type TemplateParams map[string]interface{} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 8c02fed2e..d5f9c744f 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -408,7 +408,19 @@ func (c *Cluster) labelsSet(shouldAddExtraLabels bool) labels.Set { } func (c *Cluster) labelsSelector() *metav1.LabelSelector { - return &metav1.LabelSelector{MatchLabels: c.labelsSet(false), MatchExpressions: nil} + return &metav1.LabelSelector{ + MatchLabels: c.labelsSet(false), + MatchExpressions: nil, + } +} + +func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { + return &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "connection-pool": c.connPoolName(), + }, + MatchExpressions: nil, + } } func (c *Cluster) roleLabelsSet(shouldAddExtraLabels bool, role PostgresRole) labels.Set { @@ -483,3 +495,7 @@ func (c *Cluster) GetSpec() (*acidv1.Postgresql, error) { func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } + +func (c *Cluster) needConnectionPool() bool { + return c.Spec.ConnectionPool != nil +} diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 3e6bec8db..6f071c44a 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -23,13 +23,15 @@ const fileWithNamespace = "/var/run/secrets/kubernetes.io/serviceaccount/namespa // RoleOrigin contains the code of the origin of a role type RoleOrigin int -// The rolesOrigin constant values must be sorted by the role priority for resolveNameConflict(...) to work. +// The rolesOrigin constant values must be sorted by the role priority for +// resolveNameConflict(...) to work. const ( RoleOriginUnknown RoleOrigin = iota RoleOriginManifest RoleOriginInfrastructure RoleOriginTeamsAPI RoleOriginSystem + RoleConnectionPool ) type syncUserOperation int diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index e4e429abb..a7a522566 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -83,6 +83,16 @@ type LogicalBackup struct { LogicalBackupS3SSE string `name:"logical_backup_s3_sse" default:"AES256"` } +// Operator options for connection pooler +type ConnectionPool struct { + NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` + Schema string `name:"connection_pool_schema" default:"pooler"` + User string `name:"connection_pool_user" default:"pooler"` + Type string `name:"connection_pool_type" default:"pgbouncer"` + Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` + Mode string `name:"connection_pool_mode" default:"session"` +} + // Config describes operator config type Config struct { CRD @@ -90,6 +100,7 @@ type Config struct { Auth Scalyr LogicalBackup + ConnectionPool WatchedNamespace string `name:"watched_namespace"` // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to' EtcdHost string `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS diff --git a/pkg/util/constants/roles.go b/pkg/util/constants/roles.go index 2c20d69db..3d201142c 100644 --- a/pkg/util/constants/roles.go +++ b/pkg/util/constants/roles.go @@ -2,15 +2,16 @@ package constants // Roles specific constants const ( - PasswordLength = 64 - SuperuserKeyName = "superuser" - ReplicationUserKeyName = "replication" - RoleFlagSuperuser = "SUPERUSER" - RoleFlagInherit = "INHERIT" - RoleFlagLogin = "LOGIN" - RoleFlagNoLogin = "NOLOGIN" - RoleFlagCreateRole = "CREATEROLE" - RoleFlagCreateDB = "CREATEDB" - RoleFlagReplication = "REPLICATION" - RoleFlagByPassRLS = "BYPASSRLS" + PasswordLength = 64 + SuperuserKeyName = "superuser" + ConnectionPoolUserKeyName = "pooler" + ReplicationUserKeyName = "replication" + RoleFlagSuperuser = "SUPERUSER" + RoleFlagInherit = "INHERIT" + RoleFlagLogin = "LOGIN" + RoleFlagNoLogin = "NOLOGIN" + RoleFlagCreateRole = "CREATEROLE" + RoleFlagCreateDB = "CREATEDB" + RoleFlagReplication = "REPLICATION" + RoleFlagByPassRLS = "BYPASSRLS" ) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index c7b2366b0..be9e216dd 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -39,6 +39,7 @@ type KubernetesClient struct { corev1.NamespacesGetter corev1.ServiceAccountsGetter appsv1.StatefulSetsGetter + appsv1.DeploymentsGetter rbacv1beta1.RoleBindingsGetter policyv1beta1.PodDisruptionBudgetsGetter apiextbeta1.CustomResourceDefinitionsGetter @@ -101,6 +102,7 @@ func NewFromConfig(cfg *rest.Config) (KubernetesClient, error) { kubeClient.NodesGetter = client.CoreV1() kubeClient.NamespacesGetter = client.CoreV1() kubeClient.StatefulSetsGetter = client.AppsV1() + kubeClient.DeploymentsGetter = client.AppsV1() kubeClient.PodDisruptionBudgetsGetter = client.PolicyV1beta1() kubeClient.RESTClient = client.CoreV1().RESTClient() kubeClient.RoleBindingsGetter = client.RbacV1beta1() From c028be493fbc6c71df34257ed505abd5e68d115c Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 15 Jan 2020 16:46:03 +0100 Subject: [PATCH 11/61] Improve cleaning up Set up a proper owner reference to StatefulSet, and delete with foreground policy to not leave orphans. --- pkg/cluster/k8sres.go | 56 ++++++++++++++++++++++++---------------- pkg/cluster/resources.go | 10 +++++-- 2 files changed, 42 insertions(+), 24 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 7d6b9be07..209402a20 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1804,6 +1804,26 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( return podTemplate, nil } +// Return an array of ownerReferences to make an arbitraty object dependent on +// the StatefulSet. Dependency is made on StatefulSet instead of PostgreSQL CRD +// while the former is represent the actual state, and only it's deletion means +// we delete the cluster (e.g. if CRD was deleted, StatefulSet somehow +// survived, we can't delete an object because it will affect the functioning +// cluster). +func (c *Cluster) ownerReferences() []metav1.OwnerReference { + controller := true + + return []metav1.OwnerReference{ + { + UID: c.Statefulset.ObjectMeta.UID, + APIVersion: "apps/v1", + Kind: "StatefulSet", + Name: c.Statefulset.ObjectMeta.Name, + Controller: &controller, + }, + } +} + func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { @@ -1823,17 +1843,13 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( Namespace: c.Namespace, Labels: c.labelsSet(true), Annotations: map[string]string{}, - // make Postgresql CRD object its owner, so that if CRD object is - // deleted, this object will be deleted even if something went - // wrong and operator didn't deleted it. - OwnerReferences: []metav1.OwnerReference{ - { - UID: c.Statefulset.ObjectMeta.UID, - APIVersion: "apps/v1", - Kind: "StatefulSet", - Name: c.Statefulset.ObjectMeta.Name, - }, - }, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Ophaned" + // propagation policy, which means that it's deletion will not + // clean up this deployment, but there is a hope that this object + // will be garbage collected if something went wrong and operator + // didn't deleted it. + OwnerReferences: c.ownerReferences(), }, Spec: appsv1.DeploymentSpec{ Replicas: numberOfInstances, @@ -1866,17 +1882,13 @@ func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service Namespace: c.Namespace, Labels: c.labelsSet(true), Annotations: map[string]string{}, - // make Postgresql CRD object its owner, so that if CRD object is - // deleted, this object will be deleted even if something went - // wrong and operator didn't deleted it. - OwnerReferences: []metav1.OwnerReference{ - { - UID: c.Postgresql.ObjectMeta.UID, - APIVersion: acidv1.APIVersion, - Kind: acidv1.PostgresqlKind, - Name: c.Postgresql.ObjectMeta.Name, - }, - }, + // make StatefulSet object its owner to represent the dependency. + // By itself StatefulSet is being deleted with "Ophaned" + // propagation policy, which means that it's deletion will not + // clean up this service, but there is a hope that this object will + // be garbage collected if something went wrong and operator didn't + // deleted it. + OwnerReferences: c.ownerReferences(), }, Spec: serviceSpec, } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 7baa96c02..b4c7e578f 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -154,10 +154,14 @@ func (c *Cluster) deleteConnectionPool() (err error) { return nil } + // set delete propagation policy to foreground, so that replica set will be + // also deleted. + policy := metav1.DeletePropagationForeground + options := metav1.DeleteOptions{PropagationPolicy: &policy} deployment := c.ConnectionPool.Deployment err = c.KubeClient. Deployments(deployment.Namespace). - Delete(deployment.Name, c.deleteOptions) + Delete(deployment.Name, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool deployment was already deleted") @@ -168,10 +172,12 @@ func (c *Cluster) deleteConnectionPool() (err error) { c.logger.Infof("Connection pool deployment %q has been deleted", util.NameFromMeta(deployment.ObjectMeta)) + // set delete propagation policy to foreground, so that all the dependant + // will be deleted. service := c.ConnectionPool.Service err = c.KubeClient. Services(service.Namespace). - Delete(service.Name, c.deleteOptions) + Delete(service.Name, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool service was already deleted") From 7254039f2ef41e5b6941277478130d27d615d4f8 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 20 Jan 2020 16:06:45 +0100 Subject: [PATCH 12/61] Add CRD configuration With convertion for config, and start tests. --- .../v1/operator_configuration_type.go | 15 +++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 2 + pkg/cluster/k8sres.go | 47 +++++++++++--- pkg/cluster/k8sres_test.go | 65 +++++++++++++++++++ pkg/controller/operator_config.go | 12 ++++ pkg/util/config/config.go | 16 +++-- 6 files changed, 142 insertions(+), 15 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index ded5261fb..58f171843 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -152,6 +152,20 @@ type ScalyrConfiguration struct { ScalyrMemoryLimit string `json:"scalyr_memory_limit,omitempty"` } +// Defines default configuration for connection pool +type ConnectionPoolConfiguration struct { + NumberOfInstances *int32 `json:"connection_pool_instances_number,omitempty"` + Schema string `json:"connection_pool_schema,omitempty"` + User string `json:"connection_pool_user,omitempty"` + Type string `json:"connection_pool_type,omitempty"` + Image string `json:"connection_pool_image,omitempty"` + Mode string `json:"connection_pool_mode,omitempty"` + DefaultCPURequest string `name:"connection_pool_default_cpu_request,omitempty"` + DefaultMemoryRequest string `name:"connection_pool_default_memory_request,omitempty"` + DefaultCPULimit string `name:"connection_pool_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `name:"connection_pool_default_memory_limit,omitempty"` +} + // OperatorLogicalBackupConfiguration defines configuration for logical backup type OperatorLogicalBackupConfiguration struct { Schedule string `json:"logical_backup_schedule,omitempty"` @@ -188,6 +202,7 @@ type OperatorConfigurationData struct { LoggingRESTAPI LoggingRESTAPIConfiguration `json:"logging_rest_api"` Scalyr ScalyrConfiguration `json:"scalyr"` LogicalBackup OperatorLogicalBackupConfiguration `json:"logical_backup"` + ConnectionPool ConnectionPoolConfiguration `json:"connection_pool"` } //Duration shortens this frequently used name diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index c50d9c902..e4d56c6e8 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -165,4 +165,6 @@ type ConnectionPool struct { Type *string `json:"type,omitempty"` Mode *string `json:"mode,omitempty"` PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` + + Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 209402a20..f992c2244 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -120,10 +120,39 @@ func (c *Cluster) makeDefaultResources() acidv1.Resources { config := c.OpConfig - defaultRequests := acidv1.ResourceDescription{CPU: config.DefaultCPURequest, Memory: config.DefaultMemoryRequest} - defaultLimits := acidv1.ResourceDescription{CPU: config.DefaultCPULimit, Memory: config.DefaultMemoryLimit} + defaultRequests := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPURequest, + Memory: config.Resources.DefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.Resources.DefaultCPULimit, + Memory: config.Resources.DefaultMemoryLimit, + } - return acidv1.Resources{ResourceRequests: defaultRequests, ResourceLimits: defaultLimits} + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } +} + +// Generate default resource section for connection pool deployment, to be used +// if nothing custom is specified in the manifest +func (c *Cluster) makeDefaultConnPoolResources() acidv1.Resources { + config := c.OpConfig + + defaultRequests := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPURequest, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryRequest, + } + defaultLimits := acidv1.ResourceDescription{ + CPU: config.ConnectionPool.ConnPoolDefaultCPULimit, + Memory: config.ConnectionPool.ConnPoolDefaultMemoryLimit, + } + + return acidv1.Resources{ + ResourceRequests: defaultRequests, + ResourceLimits: defaultLimits, + } } func generateResourceRequirements(resources acidv1.Resources, defaultResources acidv1.Resources) (*v1.ResourceRequirements, error) { @@ -765,12 +794,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef request := spec.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } limit := spec.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(request, limit) @@ -792,12 +821,12 @@ func (c *Cluster) generateStatefulSet(spec *acidv1.PostgresSpec) (*appsv1.Statef // TODO #413 sidecarRequest := sidecar.Resources.ResourceRequests.Memory if request == "" { - request = c.OpConfig.DefaultMemoryRequest + request = c.OpConfig.Resources.DefaultMemoryRequest } sidecarLimit := sidecar.Resources.ResourceLimits.Memory if limit == "" { - limit = c.OpConfig.DefaultMemoryLimit + limit = c.OpConfig.Resources.DefaultMemoryLimit } isSmaller, err := util.IsSmallerQuantity(sidecarRequest, sidecarLimit) @@ -1710,8 +1739,8 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( if podTemplate == nil { gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) resources, err := generateResourceRequirements( - c.Spec.Resources, - c.makeDefaultResources()) + spec.ConnectionPool.Resources, + c.makeDefaultConnPoolResources()) effectiveMode := spec.ConnectionPool.Mode if effectiveMode == nil { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index e8fe05456..aa9ef6513 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -1,6 +1,7 @@ package cluster import ( + "errors" "reflect" v1 "k8s.io/api/core/v1" @@ -451,3 +452,67 @@ func TestSecretVolume(t *testing.T) { } } } + +func TestConnPoolPodTemplate(t *testing.T) { + testName := "Test connection pool pod template generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + var clusterNoDefaultRes = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{}, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + }{ + { + subTest: "empty pod template", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + }, + { + subTest: "no default resources", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), + cluster: clusterNoDefaultRes, + }, + } + for _, tt := range tests { + _, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + } +} diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index c6f10faa0..f5d280363 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -142,5 +142,17 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit + // connection pool + result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances + result.ConnectionPool.Schema = fromCRD.ConnectionPool.Schema + result.ConnectionPool.User = fromCRD.ConnectionPool.User + result.ConnectionPool.Type = fromCRD.ConnectionPool.Type + result.ConnectionPool.Image = fromCRD.ConnectionPool.Image + result.ConnectionPool.Mode = fromCRD.ConnectionPool.Mode + result.ConnectionPool.ConnPoolDefaultCPURequest = fromCRD.ConnectionPool.DefaultCPURequest + result.ConnectionPool.ConnPoolDefaultMemoryRequest = fromCRD.ConnectionPool.DefaultMemoryRequest + result.ConnectionPool.ConnPoolDefaultCPULimit = fromCRD.ConnectionPool.DefaultCPULimit + result.ConnectionPool.ConnPoolDefaultMemoryLimit = fromCRD.ConnectionPool.DefaultMemoryLimit + return result } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index a7a522566..2baf99931 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,12 +85,16 @@ type LogicalBackup struct { // Operator options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` - Schema string `name:"connection_pool_schema" default:"pooler"` - User string `name:"connection_pool_user" default:"pooler"` - Type string `name:"connection_pool_type" default:"pgbouncer"` - Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` - Mode string `name:"connection_pool_mode" default:"session"` + NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` + Schema string `name:"connection_pool_schema" default:"pooler"` + User string `name:"connection_pool_user" default:"pooler"` + Type string `name:"connection_pool_type" default:"pgbouncer"` + Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` + Mode string `name:"connection_pool_mode" default:"session"` + ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"100m"` + ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` + ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"3"` + ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"1Gi"` } // Config describes operator config From 8bd2086cd25b52a80724f493a7cf15335d72bade Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 22 Jan 2020 14:49:54 +0100 Subject: [PATCH 13/61] Add more tests --- pkg/cluster/cluster.go | 2 +- pkg/cluster/k8sres.go | 5 + pkg/cluster/k8sres_test.go | 327 +++++++++++++++++++++++++++++++++- pkg/cluster/resources.go | 4 +- pkg/cluster/resources_test.go | 65 +++++++ pkg/cluster/types.go | 2 + pkg/util/k8sutil/k8sutil.go | 57 +++++- 7 files changed, 452 insertions(+), 10 deletions(-) create mode 100644 pkg/cluster/resources_test.go diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1681e7d2e..afcb0df82 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -336,7 +336,7 @@ func (c *Cluster) Create() error { c.logger.Warning("Connection pool already exists in the cluster") return nil } - connPool, err := c.createConnectionPool() + connPool, err := c.createConnectionPool(c.installLookupFunction) if err != nil { c.logger.Warningf("could not create connection pool: %v", err) return nil diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index f992c2244..8b7860886 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1842,6 +1842,11 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( func (c *Cluster) ownerReferences() []metav1.OwnerReference { controller := true + if c.Statefulset == nil { + c.logger.Warning("Cannot get owner reference, no statefulset") + return []metav1.OwnerReference{} + } + return []metav1.OwnerReference{ { UID: c.Statefulset.ObjectMeta.UID, diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index aa9ef6513..012df4072 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -2,6 +2,7 @@ package cluster import ( "errors" + "fmt" "reflect" v1 "k8s.io/api/core/v1" @@ -14,6 +15,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" + appsv1 "k8s.io/api/apps/v1" policyv1beta1 "k8s.io/api/policy/v1beta1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" @@ -453,7 +455,80 @@ func TestSecretVolume(t *testing.T) { } } -func TestConnPoolPodTemplate(t *testing.T) { +func testResources(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + cpuReq := podSpec.Spec.Containers[0].Resources.Requests["cpu"] + if cpuReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest { + return fmt.Errorf("CPU request doesn't match, got %s, expected %s", + cpuReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPURequest) + } + + memReq := podSpec.Spec.Containers[0].Resources.Requests["memory"] + if memReq.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest { + return fmt.Errorf("Memory request doesn't match, got %s, expected %s", + memReq.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryRequest) + } + + cpuLim := podSpec.Spec.Containers[0].Resources.Limits["cpu"] + if cpuLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit { + return fmt.Errorf("CPU limit doesn't match, got %s, expected %s", + cpuLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultCPULimit) + } + + memLim := podSpec.Spec.Containers[0].Resources.Limits["memory"] + if memLim.String() != cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit { + return fmt.Errorf("Memory limit doesn't match, got %s, expected %s", + memLim.String(), cluster.OpConfig.ConnectionPool.ConnPoolDefaultMemoryLimit) + } + + return nil +} + +func testLabels(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + poolLabels := podSpec.ObjectMeta.Labels["connection-pool"] + + if poolLabels != cluster.connPoolLabelsSelector().MatchLabels["connection-pool"] { + return fmt.Errorf("Pod labels do not match, got %+v, expected %+v", + podSpec.ObjectMeta.Labels, cluster.connPoolLabelsSelector().MatchLabels) + } + + return nil +} + +func testEnvs(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + required := map[string]bool{ + "PGHOST": false, + "PGPORT": false, + "PGUSER": false, + "PGSCHEMA": false, + "PGPASSWORD": false, + "CONNECTION_POOL_MODE": false, + "CONNECTION_POOL_PORT": false, + } + + envs := podSpec.Spec.Containers[0].Env + for _, env := range envs { + required[env.Name] = true + } + + for env, value := range required { + if !value { + return fmt.Errorf("Environment variable %s is not present", env) + } + } + + return nil +} + +func testCustomPodTemplate(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { + if podSpec.ObjectMeta.Name != "test-pod-template" { + return fmt.Errorf("Custom pod template is not used, current spec %+v", + podSpec) + } + + return nil +} + +func TestConnPoolPodSpec(t *testing.T) { testName := "Test connection pool pod template generation" var cluster = New( Config{ @@ -484,19 +559,23 @@ func TestConnPoolPodTemplate(t *testing.T) { }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + noCheck := func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error { return nil } + tests := []struct { subTest string spec *acidv1.PostgresSpec expected error cluster *Cluster + check func(cluster *Cluster, podSpec *v1.PodTemplateSpec) error }{ { - subTest: "empty pod template", + subTest: "default configuration", spec: &acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{}, }, expected: nil, cluster: cluster, + check: noCheck, }, { subTest: "no default resources", @@ -505,14 +584,256 @@ func TestConnPoolPodTemplate(t *testing.T) { }, expected: errors.New(`could not generate resource requirements: could not fill resource requests: could not parse default CPU quantity: quantities must match the regular expression '^([+-]?[0-9.]+)([eEinumkKMGTP]*[-+]?[0-9]*)$'`), cluster: clusterNoDefaultRes, + check: noCheck, + }, + { + subTest: "default resources are set", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testResources, + }, + { + subTest: "labels for service", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testLabels, + }, + { + subTest: "required envs", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testEnvs, + }, + { + subTest: "custom pod template", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + PodTemplate: &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-template", + }, + }, + }, + }, + expected: nil, + cluster: cluster, + check: testCustomPodTemplate, }, } for _, tt := range tests { - _, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) + podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) if err != tt.expected && err.Error() != tt.expected.Error() { t.Errorf("%s [%s]: Could not generate pod template,\n %+v, expected\n %+v", testName, tt.subTest, err, tt.expected) } + + err = tt.check(cluster, podSpec) + if err != nil { + t.Errorf("%s [%s]: Pod spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testDeploymentOwnwerReference(cluster *Cluster, deployment *appsv1.Deployment) error { + owner := deployment.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testSelector(cluster *Cluster, deployment *appsv1.Deployment) error { + labels := deployment.Spec.Selector.MatchLabels + expected := cluster.connPoolLabelsSelector().MatchLabels + + if labels["connection-pool"] != expected["connection-pool"] { + return fmt.Errorf("Labels are incorrect, got %+v, expected %+v", + labels, expected) + } + + return nil +} + +func TestConnPoolDeploymentSpec(t *testing.T) { + testName := "Test connection pool deployment spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *appsv1.Deployment) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + expected error + cluster *Cluster + check func(cluster *Cluster, deployment *appsv1.Deployment) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testDeploymentOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + expected: nil, + cluster: cluster, + check: testSelector, + }, + } + for _, tt := range tests { + deployment, err := tt.cluster.generateConnPoolDeployment(tt.spec) + + if err != tt.expected && err.Error() != tt.expected.Error() { + t.Errorf("%s [%s]: Could not generate deployment spec,\n %+v, expected\n %+v", + testName, tt.subTest, err, tt.expected) + } + + err = tt.check(cluster, deployment) + if err != nil { + t.Errorf("%s [%s]: Deployment spec is incorrect, %+v", + testName, tt.subTest, err) + } + } +} + +func testServiceOwnwerReference(cluster *Cluster, service *v1.Service) error { + owner := service.ObjectMeta.OwnerReferences[0] + + if owner.Name != cluster.Statefulset.ObjectMeta.Name { + return fmt.Errorf("Ownere reference is incorrect, got %s, expected %s", + owner.Name, cluster.Statefulset.ObjectMeta.Name) + } + + return nil +} + +func testServiceSelector(cluster *Cluster, service *v1.Service) error { + selector := service.Spec.Selector + + if selector["connection-pool"] != cluster.connPoolName() { + return fmt.Errorf("Selector is incorrect, got %s, expected %s", + selector["connection-pool"], cluster.connPoolName()) + } + + return nil +} + +func TestConnPoolServiceSpec(t *testing.T) { + testName := "Test connection pool service spec generation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + noCheck := func(cluster *Cluster, deployment *v1.Service) error { + return nil + } + + tests := []struct { + subTest string + spec *acidv1.PostgresSpec + cluster *Cluster + check func(cluster *Cluster, deployment *v1.Service) error + }{ + { + subTest: "default configuration", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: noCheck, + }, + { + subTest: "owner reference", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceOwnwerReference, + }, + { + subTest: "selector", + spec: &acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + cluster: cluster, + check: testServiceSelector, + }, + } + for _, tt := range tests { + service := tt.cluster.generateConnPoolService(tt.spec) + + if err := tt.check(cluster, service); err != nil { + t.Errorf("%s [%s]: Service spec is incorrect, %+v", + testName, tt.subTest, err) + } } } diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index b4c7e578f..4f9d72e19 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -97,11 +97,11 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { // // After that create all the objects for connection pool, namely a deployment // with a chosen pooler and a service to expose it. -func (c *Cluster) createConnectionPool() (*ConnectionPoolResources, error) { +func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolResources, error) { var msg string c.setProcessName("creating connection pool") - err := c.installLookupFunction( + err := lookup( c.OpConfig.ConnectionPool.Schema, c.OpConfig.ConnectionPool.User) diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go new file mode 100644 index 000000000..a3754c564 --- /dev/null +++ b/pkg/cluster/resources_test.go @@ -0,0 +1,65 @@ +package cluster + +import ( + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mockInstallLookupFunction(schema string, user string) error { + return nil +} + +func TestConnPoolCreationAndDeletion(t *testing.T) { + testName := "Test connection pool creation" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + poolResources, err := cluster.createConnectionPool(mockInstallLookupFunction) + + if err != nil { + t.Errorf("%s: Cannot create connection pool, %s, %+v", + testName, err, poolResources) + } + + if poolResources.Deployment == nil { + t.Errorf("%s: Connection pool deployment is empty", testName) + } + + if poolResources.Service == nil { + t.Errorf("%s: Connection pool service is empty", testName) + } + + err = cluster.deleteConnectionPool() + if err != nil { + t.Errorf("%s: Cannot delete connection pool, %s", testName, err) + } +} diff --git a/pkg/cluster/types.go b/pkg/cluster/types.go index 286505621..04d00cb58 100644 --- a/pkg/cluster/types.go +++ b/pkg/cluster/types.go @@ -71,3 +71,5 @@ type ClusterStatus struct { } type TemplateParams map[string]interface{} + +type InstallFunction func(schema string, user string) error diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index be9e216dd..fed85d1d6 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -9,6 +9,7 @@ import ( batchv1beta1 "k8s.io/api/batch/v1beta1" clientbatchv1beta1 "k8s.io/client-go/kubernetes/typed/batch/v1beta1" + apiappsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" policybeta1 "k8s.io/api/policy/v1beta1" apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" @@ -56,6 +57,20 @@ type mockSecret struct { type MockSecretGetter struct { } +type mockDeployment struct { + appsv1.DeploymentInterface +} + +type MockDeploymentGetter struct { +} + +type mockService struct { + corev1.ServiceInterface +} + +type MockServiceGetter struct { +} + type mockConfigMap struct { corev1.ConfigMapInterface } @@ -232,19 +247,53 @@ func (c *mockConfigMap) Get(name string, options metav1.GetOptions) (*v1.ConfigM } // Secrets to be mocked -func (c *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { +func (mock *MockSecretGetter) Secrets(namespace string) corev1.SecretInterface { return &mockSecret{} } // ConfigMaps to be mocked -func (c *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { +func (mock *MockConfigMapsGetter) ConfigMaps(namespace string) corev1.ConfigMapInterface { return &mockConfigMap{} } +func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeployment{} +} + +func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + +func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface { + return &mockService{} +} + +func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { + return nil +} + // NewMockKubernetesClient for other tests func NewMockKubernetesClient() KubernetesClient { return KubernetesClient{ - SecretsGetter: &MockSecretGetter{}, - ConfigMapsGetter: &MockConfigMapsGetter{}, + SecretsGetter: &MockSecretGetter{}, + ConfigMapsGetter: &MockConfigMapsGetter{}, + DeploymentsGetter: &MockDeploymentGetter{}, + ServicesGetter: &MockServiceGetter{}, } } From 3ff1147bcefafc89dca084e17116f638a9923508 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 12 Feb 2020 17:28:48 +0100 Subject: [PATCH 14/61] Various improvements Add synchronization logic. For now get rid of podTemplate, type fields. Add crd validation & configuration part, put retry on top of lookup function installation. --- .../templates/clusterrole.yaml | 1 + manifests/operatorconfiguration.crd.yaml | 37 ++++ pkg/apis/acid.zalan.do/v1/postgresql_type.go | 18 +- pkg/cluster/cluster.go | 44 ++++- pkg/cluster/database.go | 22 ++- pkg/cluster/k8sres.go | 163 +++++++++--------- pkg/cluster/k8sres_test.go | 15 -- pkg/cluster/resources.go | 75 +++++++- pkg/cluster/sync.go | 111 ++++++++++++ pkg/cluster/sync_test.go | 125 ++++++++++++++ pkg/cluster/util.go | 2 +- pkg/controller/operator_config.go | 57 ++++-- pkg/util/constants/pooler.go | 13 ++ pkg/util/k8sutil/k8sutil.go | 93 ++++++++++ 14 files changed, 647 insertions(+), 129 deletions(-) create mode 100644 pkg/cluster/sync_test.go create mode 100644 pkg/util/constants/pooler.go diff --git a/charts/postgres-operator/templates/clusterrole.yaml b/charts/postgres-operator/templates/clusterrole.yaml index f8550a539..316f7de15 100644 --- a/charts/postgres-operator/templates/clusterrole.yaml +++ b/charts/postgres-operator/templates/clusterrole.yaml @@ -106,6 +106,7 @@ rules: - apps resources: - statefulsets + - deployments verbs: - create - delete diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 7bd5c529c..c44955771 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -294,6 +294,43 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" status: type: object additionalProperties: diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index e4d56c6e8..c56d70626 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,7 +27,8 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` + EnableConnectionPool bool `json:"enable_connection_pool,omitempty"` + ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` TeamID string `json:"teamId"` DockerImage string `json:"dockerImage,omitempty"` @@ -159,12 +160,15 @@ type PostgresStatus struct { // Options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `json:"instancesNumber,omitempty"` - Schema *string `json:"schema,omitempty"` - User *string `json:"user,omitempty"` - Type *string `json:"type,omitempty"` - Mode *string `json:"mode,omitempty"` - PodTemplate *v1.PodTemplateSpec `json:"podTemplate,omitempty"` + NumberOfInstances *int32 `json:"instancesNumber,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` + // TODO: prepared snippets of configuration, one can choose via type, e.g. + // pgbouncer-large (with higher resources) or odyssey-small (with smaller + // resources) + // Type string `json:"type,omitempty"` Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index afcb0df82..8e65b12ea 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -11,6 +11,7 @@ import ( "sync" "time" + "github.com/r3labs/diff" "github.com/sirupsen/logrus" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -723,6 +724,17 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } + // connection pool + if !reflect.DeepEqual(oldSpec.Spec.ConnectionPool, + newSpec.Spec.ConnectionPool) { + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPool(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + updateFailed = true + } + } + return nil } @@ -852,13 +864,13 @@ func (c *Cluster) initSystemUsers() { if c.needConnectionPool() { username := c.Spec.ConnectionPool.User - if username == nil { - username = &c.OpConfig.ConnectionPool.User + if username == "" { + username = c.OpConfig.ConnectionPool.User } c.systemUsers[constants.ConnectionPoolUserKeyName] = spec.PgUser{ Origin: spec.RoleConnectionPool, - Name: *username, + Name: username, Password: util.RandomPassword(constants.PasswordLength), } } @@ -1188,3 +1200,29 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { return c.deleteClusterObject(get, deleteConfigMapFn, "configmap") } + +// Test if two connection pool configuration needs to be synced. For simplicity +// compare not the actual K8S objects, but the configuration itself and request +// sync if there is any difference. +func (c *Cluster) needSyncConnPoolDeployments(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { + reasons = []string{} + sync = false + + changelog, err := diff.Diff(oldSpec, newSpec) + if err != nil { + c.logger.Infof("Cannot get diff, do not do anything, %+v", err) + return false, reasons + } else { + if len(changelog) > 0 { + sync = true + } + + for _, change := range changelog { + msg := fmt.Sprintf("%s %+v from %s to %s", + change.Type, change.Path, change.From, change.To) + reasons = append(reasons, msg) + } + } + + return sync, reasons +} diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 1b74bd6b6..0c1e07a11 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -300,8 +300,26 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { params, err) } - if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { - return fmt.Errorf("could not execute sql statement %s: %v", + // golang sql will do retries couple of times if pq driver reports + // connections issues (driver.ErrBadConn), but since our query is + // idempotent, we can retry in a view of other errors (e.g. due to + // failover a db is temporary in a read-only mode or so) to make sure + // it was applied. + execErr := retryutil.Retry( + constants.PostgresConnectTimeout, + constants.PostgresConnectRetryTimeout, + func() (bool, error) { + if _, err := c.pgDb.Exec(stmtBytes.String()); err != nil { + msg := fmt.Errorf("could not execute sql statement %s: %v", + stmtBytes.String(), err) + return false, msg + } + + return true, nil + }) + + if execErr != nil { + return fmt.Errorf("could not execute after retries %s: %v", stmtBytes.String(), err) } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 8b7860886..3121691d9 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1734,100 +1734,99 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( *v1.PodTemplateSpec, error) { - podTemplate := spec.ConnectionPool.PodTemplate + gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) + resources, err := generateResourceRequirements( + spec.ConnectionPool.Resources, + c.makeDefaultConnPoolResources()) - if podTemplate == nil { - gracePeriod := int64(c.OpConfig.PodTerminateGracePeriod.Seconds()) - resources, err := generateResourceRequirements( - spec.ConnectionPool.Resources, - c.makeDefaultConnPoolResources()) + effectiveMode := util.Coalesce( + spec.ConnectionPool.Mode, + c.OpConfig.ConnectionPool.Mode) - effectiveMode := spec.ConnectionPool.Mode - if effectiveMode == nil { - effectiveMode = &c.OpConfig.ConnectionPool.Mode - } + effectiveDockerImage := util.Coalesce( + spec.ConnectionPool.DockerImage, + c.OpConfig.ConnectionPool.Image) - if err != nil { - return nil, fmt.Errorf("could not generate resource requirements: %v", err) - } + if err != nil { + return nil, fmt.Errorf("could not generate resource requirements: %v", err) + } - secretSelector := func(key string) *v1.SecretKeySelector { - return &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: c.credentialSecretName(c.OpConfig.SuperUsername), - }, - Key: key, - } + secretSelector := func(key string) *v1.SecretKeySelector { + return &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: c.credentialSecretName(c.OpConfig.SuperUsername), + }, + Key: key, } + } - envVars := []v1.EnvVar{ - { - Name: "PGHOST", - Value: c.serviceAddress(Master), - }, - { - Name: "PGPORT", - Value: c.servicePort(Master), - }, - { - Name: "PGUSER", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("username"), - }, - }, - // the convention is to use the same schema name as - // connection pool username - { - Name: "PGSCHEMA", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("username"), - }, - }, - { - Name: "PGPASSWORD", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("password"), - }, + envVars := []v1.EnvVar{ + { + Name: "PGHOST", + Value: c.serviceAddress(Master), + }, + { + Name: "PGPORT", + Value: c.servicePort(Master), + }, + { + Name: "PGUSER", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), }, - { - Name: "CONNECTION_POOL_MODE", - Value: *effectiveMode, + }, + // the convention is to use the same schema name as + // connection pool username + { + Name: "PGSCHEMA", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("username"), }, - { - Name: "CONNECTION_POOL_PORT", - Value: fmt.Sprint(pgPort), + }, + { + Name: "PGPASSWORD", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: secretSelector("password"), }, - } + }, + { + Name: "CONNECTION_POOL_MODE", + Value: effectiveMode, + }, + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + } - poolerContainer := v1.Container{ - Name: connectionPoolContainer, - Image: c.OpConfig.ConnectionPool.Image, - ImagePullPolicy: v1.PullIfNotPresent, - Resources: *resources, - Ports: []v1.ContainerPort{ - { - ContainerPort: pgPort, - Protocol: v1.ProtocolTCP, - }, + poolerContainer := v1.Container{ + Name: connectionPoolContainer, + Image: effectiveDockerImage, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: *resources, + Ports: []v1.ContainerPort{ + { + ContainerPort: pgPort, + Protocol: v1.ProtocolTCP, }, - Env: envVars, - } + }, + Env: envVars, + } - podTemplate = &v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: c.connPoolLabelsSelector().MatchLabels, - Namespace: c.Namespace, - Annotations: c.generatePodAnnotations(spec), - }, - Spec: v1.PodSpec{ - ServiceAccountName: c.OpConfig.PodServiceAccountName, - TerminationGracePeriodSeconds: &gracePeriod, - Containers: []v1.Container{poolerContainer}, - // TODO: add tolerations to scheduler pooler on the same node - // as database - //Tolerations: *tolerationsSpec, - }, - } + podTemplate := &v1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: c.connPoolLabelsSelector().MatchLabels, + Namespace: c.Namespace, + Annotations: c.generatePodAnnotations(spec), + }, + Spec: v1.PodSpec{ + ServiceAccountName: c.OpConfig.PodServiceAccountName, + TerminationGracePeriodSeconds: &gracePeriod, + Containers: []v1.Container{poolerContainer}, + // TODO: add tolerations to scheduler pooler on the same node + // as database + //Tolerations: *tolerationsSpec, + }, } return podTemplate, nil diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 012df4072..c26e04b96 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -613,21 +613,6 @@ func TestConnPoolPodSpec(t *testing.T) { cluster: cluster, check: testEnvs, }, - { - subTest: "custom pod template", - spec: &acidv1.PostgresSpec{ - ConnectionPool: &acidv1.ConnectionPool{ - PodTemplate: &v1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-pod-template", - }, - }, - }, - }, - expected: nil, - cluster: cluster, - check: testCustomPodTemplate, - }, } for _, tt := range tests { podSpec, err := tt.cluster.generateConnPoolPodTemplate(tt.spec) diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 4f9d72e19..e44d50800 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -101,9 +101,17 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolR var msg string c.setProcessName("creating connection pool") - err := lookup( - c.OpConfig.ConnectionPool.Schema, - c.OpConfig.ConnectionPool.User) + schema := c.Spec.ConnectionPool.Schema + if schema == "" { + schema = c.OpConfig.ConnectionPool.Schema + } + + user := c.Spec.ConnectionPool.User + if user == "" { + user = c.OpConfig.ConnectionPool.User + } + + err := lookup(schema, user) if err != nil { msg = "could not prepare database for connection pool: %v" @@ -116,6 +124,9 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolR return nil, fmt.Errorf(msg, err) } + // client-go does retry 10 times (with NoBackoff by default) when the API + // believe a request can be retried and returns Retry-After header. This + // should be good enough to not think about it here. deployment, err := c.KubeClient. Deployments(deploymentSpec.Namespace). Create(deploymentSpec) @@ -154,14 +165,22 @@ func (c *Cluster) deleteConnectionPool() (err error) { return nil } + // Clean up the deployment object. If deployment resource we've remembered + // is somehow empty, try to delete based on what would we generate + deploymentName := c.connPoolName() + deployment := c.ConnectionPool.Deployment + + if deployment != nil { + deploymentName = deployment.Name + } + // set delete propagation policy to foreground, so that replica set will be // also deleted. policy := metav1.DeletePropagationForeground options := metav1.DeleteOptions{PropagationPolicy: &policy} - deployment := c.ConnectionPool.Deployment err = c.KubeClient. - Deployments(deployment.Namespace). - Delete(deployment.Name, &options) + Deployments(c.Namespace). + Delete(deploymentName, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool deployment was already deleted") @@ -172,12 +191,19 @@ func (c *Cluster) deleteConnectionPool() (err error) { c.logger.Infof("Connection pool deployment %q has been deleted", util.NameFromMeta(deployment.ObjectMeta)) + // Repeat the same for the service object + service := c.ConnectionPool.Service + serviceName := c.connPoolName() + + if service != nil { + serviceName = service.Name + } + // set delete propagation policy to foreground, so that all the dependant // will be deleted. - service := c.ConnectionPool.Service err = c.KubeClient. - Services(service.Namespace). - Delete(service.Name, &options) + Services(c.Namespace). + Delete(serviceName, &options) if !k8sutil.ResourceNotFound(err) { c.logger.Debugf("Connection pool service was already deleted") @@ -823,3 +849,34 @@ func (c *Cluster) GetStatefulSet() *appsv1.StatefulSet { func (c *Cluster) GetPodDisruptionBudget() *policybeta1.PodDisruptionBudget { return c.PodDisruptionBudget } + +// Perform actual patching of a connection pool deployment, assuming that all +// the check were already done before. +func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *appsv1.Deployment) (*appsv1.Deployment, error) { + c.setProcessName("updating connection pool") + if c.ConnectionPool == nil || c.ConnectionPool.Deployment == nil { + return nil, fmt.Errorf("there is no connection pool in the cluster") + } + + patchData, err := specPatch(newDeployment.Spec) + if err != nil { + return nil, fmt.Errorf("could not form patch for the deployment: %v", err) + } + + // An update probably requires RetryOnConflict, but since only one operator + // worker at one time will try to update it changes of conflicts are + // minimal. + deployment, err := c.KubeClient. + Deployments(c.ConnectionPool.Deployment.Namespace). + Patch( + c.ConnectionPool.Deployment.Name, + types.MergePatchType, + patchData, "") + if err != nil { + return nil, fmt.Errorf("could not patch deployment: %v", err) + } + + c.ConnectionPool.Deployment = deployment + + return deployment, nil +} diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 5ee827d4b..b59bf5533 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -2,6 +2,7 @@ package cluster import ( "fmt" + "reflect" batchv1beta1 "k8s.io/api/batch/v1beta1" v1 "k8s.io/api/core/v1" @@ -23,6 +24,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { c.mu.Lock() defer c.mu.Unlock() + oldSpec := c.Postgresql c.setSpec(newSpec) defer func() { @@ -108,6 +110,20 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } + // connection pool + oldPool := oldSpec.Spec.ConnectionPool + newPool := newSpec.Spec.ConnectionPool + if c.needConnectionPool() && + (c.ConnectionPool == nil || !reflect.DeepEqual(oldPool, newPool)) { + + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPool(&oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } + return err } @@ -594,3 +610,98 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } + +// Synchronize connection pool resources. Effectively we're interested only in +// synchronizing the corresponding deployment, but in case of deployment or +// service is missing, create it. After checking, also remember an object for +// the future references. +func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error { + if c.ConnectionPool == nil { + c.logger.Warning("Connection pool resources are empty") + c.ConnectionPool = &ConnectionPoolResources{} + } + + deployment, err := c.KubeClient. + Deployments(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Deployment %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + deploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg = "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + deployment, err := c.KubeClient. + Deployments(deploymentSpec.Namespace). + Create(deploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + } else if err != nil { + return fmt.Errorf("could not get connection pool deployment to sync: %v", err) + } else { + c.ConnectionPool.Deployment = deployment + + // actual synchronization + oldConnPool := oldSpec.Spec.ConnectionPool + newConnPool := newSpec.Spec.ConnectionPool + sync, reason := c.needSyncConnPoolDeployments(oldConnPool, newConnPool) + if sync { + c.logger.Infof("Update connection pool deployment %s, reason: %s", + c.connPoolName(), reason) + + newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) + if err != nil { + msg := "could not generate deployment for connection pool: %v" + return fmt.Errorf(msg, err) + } + + oldDeploymentSpec := c.ConnectionPool.Deployment + + deployment, err := c.updateConnPoolDeployment( + oldDeploymentSpec, + newDeploymentSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Deployment = deployment + return nil + } + } + + service, err := c.KubeClient. + Services(c.Namespace). + Get(c.connPoolName(), metav1.GetOptions{}) + + if err != nil && k8sutil.ResourceNotFound(err) { + msg := "Service %s for connection pool synchronization is not found, create it" + c.logger.Warningf(msg, c.connPoolName()) + + serviceSpec := c.generateConnPoolService(&newSpec.Spec) + service, err := c.KubeClient. + Services(serviceSpec.Namespace). + Create(serviceSpec) + + if err != nil { + return err + } + + c.ConnectionPool.Service = service + } else if err != nil { + return fmt.Errorf("could not get connection pool service to sync: %v", err) + } else { + // Service updates are not supported and probably not that useful anyway + c.ConnectionPool.Service = service + } + + return nil +} diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go new file mode 100644 index 000000000..c6928a64e --- /dev/null +++ b/pkg/cluster/sync_test.go @@ -0,0 +1,125 @@ +package cluster + +import ( + "fmt" + "testing" + + acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" + + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func int32ToPointer(value int32) *int32 { + return &value +} + +func deploymentUpdated(cluster *Cluster, err error) error { + if cluster.ConnectionPool.Deployment.Spec.Replicas == nil || + *cluster.ConnectionPool.Deployment.Spec.Replicas != 2 { + return fmt.Errorf("Wrong nubmer of instances") + } + + return nil +} + +func objectsAreSaved(cluster *Cluster, err error) error { + if cluster.ConnectionPool == nil { + return fmt.Errorf("Connection pool resources are empty") + } + + if cluster.ConnectionPool.Deployment == nil { + return fmt.Errorf("Deployment was not saved") + } + + if cluster.ConnectionPool.Service == nil { + return fmt.Errorf("Service was not saved") + } + + return nil +} + +func TestConnPoolSynchronization(t *testing.T) { + testName := "Test connection pool synchronization" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) + + cluster.Statefulset = &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-sts", + }, + } + + clusterMissingObjects := *cluster + clusterMissingObjects.KubeClient = k8sutil.ClientMissingObjects() + + clusterMock := *cluster + clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() + + tests := []struct { + subTest string + oldSpec *acidv1.Postgresql + newSpec *acidv1.Postgresql + cluster *Cluster + check func(cluster *Cluster, err error) error + }{ + { + subTest: "create if doesn't exist", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "update deployment", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(1), + }, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{ + NumberOfInstances: int32ToPointer(2), + }, + }, + }, + cluster: &clusterMock, + check: deploymentUpdated, + }, + } + for _, tt := range tests { + err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec) + + if err := tt.check(tt.cluster, err); err != nil { + t.Errorf("%s [%s]: Could not synchronize, %+v", + testName, tt.subTest, err) + } + } +} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index d5f9c744f..d2dd11586 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -497,5 +497,5 @@ func (c *Cluster) patroniUsesKubernetes() bool { } func (c *Cluster) needConnectionPool() bool { - return c.Spec.ConnectionPool != nil + return c.Spec.ConnectionPool != nil || c.Spec.EnableConnectionPool == true } diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index f5d280363..1748fbd1f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -6,7 +6,9 @@ import ( "time" acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" + "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -142,17 +144,52 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ScalyrCPULimit = fromCRD.Scalyr.ScalyrCPULimit result.ScalyrMemoryLimit = fromCRD.Scalyr.ScalyrMemoryLimit - // connection pool + // Connection pool. Looks like we can't use defaulting in CRD before 1.17, + // so ensure default values here. result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances - result.ConnectionPool.Schema = fromCRD.ConnectionPool.Schema - result.ConnectionPool.User = fromCRD.ConnectionPool.User - result.ConnectionPool.Type = fromCRD.ConnectionPool.Type - result.ConnectionPool.Image = fromCRD.ConnectionPool.Image - result.ConnectionPool.Mode = fromCRD.ConnectionPool.Mode - result.ConnectionPool.ConnPoolDefaultCPURequest = fromCRD.ConnectionPool.DefaultCPURequest - result.ConnectionPool.ConnPoolDefaultMemoryRequest = fromCRD.ConnectionPool.DefaultMemoryRequest - result.ConnectionPool.ConnPoolDefaultCPULimit = fromCRD.ConnectionPool.DefaultCPULimit - result.ConnectionPool.ConnPoolDefaultMemoryLimit = fromCRD.ConnectionPool.DefaultMemoryLimit + if result.ConnectionPool.NumberOfInstances == nil || + *result.ConnectionPool.NumberOfInstances < 1 { + var value int32 + + value = 1 + result.ConnectionPool.NumberOfInstances = &value + } + + result.ConnectionPool.Schema = util.Coalesce( + fromCRD.ConnectionPool.Schema, + constants.ConnectionPoolSchemaName) + + result.ConnectionPool.User = util.Coalesce( + fromCRD.ConnectionPool.User, + constants.ConnectionPoolUserName) + + result.ConnectionPool.Type = util.Coalesce( + fromCRD.ConnectionPool.Type, + constants.ConnectionPoolDefaultType) + + result.ConnectionPool.Image = util.Coalesce( + fromCRD.ConnectionPool.Image, + "pgbouncer:0.0.1") + + result.ConnectionPool.Mode = util.Coalesce( + fromCRD.ConnectionPool.Mode, + constants.ConnectionPoolDefaultMode) + + result.ConnectionPool.ConnPoolDefaultCPURequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPURequest, + constants.ConnectionPoolDefaultCpuRequest) + + result.ConnectionPool.ConnPoolDefaultMemoryRequest = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryRequest, + constants.ConnectionPoolDefaultMemoryRequest) + + result.ConnectionPool.ConnPoolDefaultCPULimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultCPULimit, + constants.ConnectionPoolDefaultCpuLimit) + + result.ConnectionPool.ConnPoolDefaultMemoryLimit = util.Coalesce( + fromCRD.ConnectionPool.DefaultMemoryLimit, + constants.ConnectionPoolDefaultMemoryLimit) return result } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go new file mode 100644 index 000000000..b25a12a6c --- /dev/null +++ b/pkg/util/constants/pooler.go @@ -0,0 +1,13 @@ +package constants + +// Connection pool specific constants +const ( + ConnectionPoolUserName = "pooler" + ConnectionPoolSchemaName = "pooler" + ConnectionPoolDefaultType = "pgbouncer" + ConnectionPoolDefaultMode = "transition" + ConnectionPoolDefaultCpuRequest = "100m" + ConnectionPoolDefaultCpuLimit = "100m" + ConnectionPoolDefaultMemoryRequest = "100M" + ConnectionPoolDefaultMemoryLimit = "100M" +) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index fed85d1d6..a58261167 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -15,6 +15,7 @@ import ( apiextclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" apiextbeta1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" appsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" @@ -27,6 +28,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +func int32ToPointer(value int32) *int32 { + return &value +} + // KubernetesClient describes getters for Kubernetes objects type KubernetesClient struct { corev1.SecretsGetter @@ -61,16 +66,30 @@ type mockDeployment struct { appsv1.DeploymentInterface } +type mockDeploymentNotExist struct { + appsv1.DeploymentInterface +} + type MockDeploymentGetter struct { } +type MockDeploymentNotExistGetter struct { +} + type mockService struct { corev1.ServiceInterface } +type mockServiceNotExist struct { + corev1.ServiceInterface +} + type MockServiceGetter struct { } +type MockServiceNotExistGetter struct { +} + type mockConfigMap struct { corev1.ConfigMapInterface } @@ -260,6 +279,10 @@ func (mock *MockDeploymentGetter) Deployments(namespace string) appsv1.Deploymen return &mockDeployment{} } +func (mock *MockDeploymentNotExistGetter) Deployments(namespace string) appsv1.DeploymentInterface { + return &mockDeploymentNotExist{} +} + func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -272,10 +295,49 @@ func (mock *mockDeployment) Delete(name string, opts *metav1.DeleteOptions) erro return nil } +func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + Spec: apiappsv1.DeploymentSpec{ + Replicas: int32ToPointer(2), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + +func (mock *mockDeploymentNotExist) Get(name string, opts metav1.GetOptions) (*apiappsv1.Deployment, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + +func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment, error) { + return &apiappsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-deployment", + }, + }, nil +} + func (mock *MockServiceGetter) Services(namespace string) corev1.ServiceInterface { return &mockService{} } +func (mock *MockServiceNotExistGetter) Services(namespace string) corev1.ServiceInterface { + return &mockServiceNotExist{} +} + func (mock *mockService) Create(*v1.Service) (*v1.Service, error) { return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -288,6 +350,30 @@ func (mock *mockService) Delete(name string, opts *metav1.DeleteOptions) error { return nil } +func (mock *mockService) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Create(*v1.Service) (*v1.Service, error) { + return &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-service", + }, + }, nil +} + +func (mock *mockServiceNotExist) Get(name string, opts metav1.GetOptions) (*v1.Service, error) { + return nil, &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Reason: metav1.StatusReasonNotFound, + }, + } +} + // NewMockKubernetesClient for other tests func NewMockKubernetesClient() KubernetesClient { return KubernetesClient{ @@ -297,3 +383,10 @@ func NewMockKubernetesClient() KubernetesClient { ServicesGetter: &MockServiceGetter{}, } } + +func ClientMissingObjects() KubernetesClient { + return KubernetesClient{ + DeploymentsGetter: &MockDeploymentNotExistGetter{}, + ServicesGetter: &MockServiceNotExistGetter{}, + } +} From 82e9d40587324b01ecd8ca7aca7b1b39d7e091eb Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 11:04:11 +0100 Subject: [PATCH 15/61] Add test for both ways to enable connection pool --- pkg/cluster/resources_test.go | 38 +++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index a3754c564..7c52addad 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -63,3 +63,41 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { t.Errorf("%s: Cannot delete connection pool, %s", testName, err) } } + +func TestNeedConnPool(t *testing.T) { + testName := "Test how connection pool can be enabled" + var cluster = New( + Config{ + OpConfig: config.Config{ + ProtectedRoles: []string{"admin"}, + Auth: config.Auth{ + SuperUsername: superUserName, + ReplicationUsername: replicationUserName, + }, + ConnectionPool: config.ConnectionPool{ + ConnPoolDefaultCPURequest: "100m", + ConnPoolDefaultCPULimit: "100m", + ConnPoolDefaultMemoryRequest: "100M", + ConnPoolDefaultMemoryLimit: "100M", + }, + }, + }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) + + cluster.Spec = acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with full definition", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: true, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag", + testName) + } +} From 17d077e3709d8681ce5e0e4ba714ce4fa66a9254 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 11:56:05 +0100 Subject: [PATCH 16/61] add validation for postgresql CRD --- .../crds/operatorconfigurations.yaml | 37 ++++++++++++++++ .../postgres-operator/crds/postgresqls.yaml | 44 +++++++++++++++++++ manifests/postgresql.crd.yaml | 44 +++++++++++++++++++ 3 files changed, 125 insertions(+) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 9725c2708..c99d4a811 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -318,6 +318,43 @@ spec: pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' scalyr_server_url: type: string + connection_pool: + type: object + properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_default_cpu_limit: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_cpu_request: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + #default: "1" + connection_pool_default_memory_limit: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" + connection_pool_default_memory_request: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + #default: "100m" status: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index b4b676236..8b7de363c 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -106,6 +106,50 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + mode: + type: string + numberOfInstances: + type: integer + minimum: 1 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 276bc94b8..7d4bb228b 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -70,6 +70,50 @@ spec: uid: format: uuid type: string + connectionPool: + type: object + properties: + dockerImage: + type: string + mode: + type: string + numberOfInstances: + type: integer + minimum: 1 + resources: + type: object + required: + - requests + - limits + properties: + limits: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + requests: + type: object + required: + - cpu + - memory + properties: + cpu: + type: string + pattern: '^(\d+m|\d+(\.\d{1,3})?)$' + memory: + type: string + pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' + schema: + type: string + user: + type: string databases: type: object additionalProperties: From f0ceafa81e32e5369d64bed59d2506840649ee76 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 12:54:32 +0100 Subject: [PATCH 17/61] reflect connectionPool validation in Go code and publish in manifests and chart --- .../crds/operatorconfigurations.yaml | 38 +++---- .../templates/configmap.yaml | 1 + .../templates/operatorconfiguration.yaml | 2 + charts/postgres-operator/values-crd.yaml | 11 ++ charts/postgres-operator/values.yaml | 12 ++ manifests/configmap.yaml | 9 ++ manifests/operatorconfiguration.crd.yaml | 38 +++---- ...gresql-operator-default-configuration.yaml | 10 ++ pkg/apis/acid.zalan.do/v1/crds.go | 104 ++++++++++++++++++ 9 files changed, 187 insertions(+), 38 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c99d4a811..a790125d1 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -321,24 +321,6 @@ spec: connection_pool: type: object properties: - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -354,7 +336,25 @@ spec: connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100m" + #default: "100Mi" + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" status: type: object additionalProperties: diff --git a/charts/postgres-operator/templates/configmap.yaml b/charts/postgres-operator/templates/configmap.yaml index 95eeb9546..634339795 100644 --- a/charts/postgres-operator/templates/configmap.yaml +++ b/charts/postgres-operator/templates/configmap.yaml @@ -20,4 +20,5 @@ data: {{ toYaml .Values.configDebug | indent 2 }} {{ toYaml .Values.configLoggingRestApi | indent 2 }} {{ toYaml .Values.configTeamsApi | indent 2 }} +{{ toYaml .Values.configConnectionPool | indent 2 }} {{- end }} diff --git a/charts/postgres-operator/templates/operatorconfiguration.yaml b/charts/postgres-operator/templates/operatorconfiguration.yaml index 6a301c1fb..55eb8fd4f 100644 --- a/charts/postgres-operator/templates/operatorconfiguration.yaml +++ b/charts/postgres-operator/templates/operatorconfiguration.yaml @@ -34,4 +34,6 @@ configuration: {{ toYaml .Values.configLoggingRestApi | indent 4 }} scalyr: {{ toYaml .Values.configScalyr | indent 4 }} + connection_pool: +{{ toYaml .Values.configConnectionPool | indent 4 }} {{- end }} diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 1f9b5e495..17b62226a 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -261,6 +261,17 @@ configScalyr: # Memory request value for the Scalyr sidecar scalyr_memory_request: 50Mi +configConnectionPool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "1" + connection_pool_default_memory_limit: 100m + connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + connection_pool_instances_number: 1 + connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 1be5851d2..f11619c8a 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -237,6 +237,18 @@ configTeamsApi: # URL of the Teams API service # teams_api_url: http://fake-teams-api.default.svc.cluster.local +# configure connection pooler deployment created by the operator +configConnectionPool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "1" + connection_pool_default_memory_limit: 100m + connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + connection_pool_instances_number: 1 + connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" + rbac: # Specifies whether RBAC resources should be created create: true diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index d26c83edf..05f22a388 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -11,6 +11,15 @@ data: cluster_history_entries: "1000" cluster_labels: application:spilo cluster_name_label: version + # connection_pool_default_cpu_limit: "1" + # connection_pool_default_cpu_request: "1" + # connection_pool_default_memory_limit: 100m + # connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + # connection_pool_instances_number: 1 + # connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" # custom_pod_annotations: "keya:valuea,keyb:valueb" db_hosted_zone: db.example.com diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index c44955771..f4224244d 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -297,24 +297,6 @@ spec: connection_pool: type: object properties: - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -330,7 +312,25 @@ spec: connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100m" + #default: "100Mi" + connection_pool_image: + type: string + #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" status: type: object additionalProperties: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index efd1a5396..037ae5e35 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -121,3 +121,13 @@ configuration: scalyr_memory_limit: 500Mi scalyr_memory_request: 50Mi # scalyr_server_url: "" + connection_pool: + connection_pool_default_cpu_limit: "1" + connection_pool_default_cpu_request: "1" + connection_pool_default_memory_limit: 100m + connection_pool_default_memory_request: "100Mi" + # connection_pool_image: "" + connection_pool_instances_number: 1 + connection_pool_mode: "transaction" + # connection_pool_schema: "pooler" + # connection_pool_user: "pooler" diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 4cfc9a9e6..f760d63e5 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -176,6 +176,65 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, }, }, + "connectionPool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "dockerImage": { + Type: "string", + }, + "mode": { + Type: "string", + }, + "numberOfInstances": { + Type: "integer", + Minimum: &min1, + }, + "resources": { + Type: "object", + Required: []string{"requests", "limits"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "limits": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + "requests": { + Type: "object", + Required: []string{"cpu", "memory"}, + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "cpu": { + Type: "string", + Description: "Decimal natural followed by m, or decimal natural followed by dot followed by up to three decimal digits (precision used by Kubernetes). Must be greater than 0", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "memory": { + Type: "string", + Description: "Plain integer or fixed-point integer using one of these suffixes: E, P, T, G, M, k (with or without a tailing i). Must be greater than 0", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + }, + }, + }, + }, + "schema": { + Type: "string", + }, + "user": { + Type: "string", + }, + }, + }, "databases": { Type: "object", AdditionalProperties: &apiextv1beta1.JSONSchemaPropsOrBool{ @@ -1037,6 +1096,51 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "connection_pool": { + Type: "object", + Properties: map[string]apiextv1beta1.JSONSchemaProps{ + "connection_pool_default_cpu_limit": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_cpu_request": { + Type: "string", + Pattern: "^(\\d+m|\\d+(\\.\\d{1,3})?)$", + }, + "connection_pool_default_memory_limit": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_default_memory_request": { + Type: "string", + Pattern: "^(\\d+(e\\d+)?|\\d+(\\.\\d+)?(e\\d+)?[EPTGMK]i?)$", + }, + "connection_pool_image": { + Type: "string", + }, + "connection_pool_instances_number": { + Type: "integer", + Minimum: &min1, + }, + "connection_pool_mode": { + Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, + }, + "connection_pool_schema": { + Type: "string", + }, + "connection_pool_user": { + Type: "string", + }, + }, + }, }, }, "status": { From 2384e1ec10638e1989c0295ffb7fbe96af2e2f1a Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 13:21:20 +0100 Subject: [PATCH 18/61] Cleanup configuration Add pool configuration into CRD & charts. Add preliminary documentation. Rename NumberOfInstances to Replicas like in Deployment. Mention couple of potential improvement points for connection pool specification. --- .../crds/operatorconfigurations.yaml | 38 ++++++------- .../postgres-operator/crds/postgresqls.yaml | 16 ++++++ charts/postgres-operator/values-crd.yaml | 24 +++++---- charts/postgres-operator/values.yaml | 22 +++++--- docs/reference/cluster_manifest.md | 27 ++++++++++ docs/reference/operator_parameters.md | 28 ++++++++++ docs/user.md | 53 +++++++++++++++++++ manifests/operatorconfiguration.crd.yaml | 36 ++++++------- manifests/postgresql.crd.yaml | 16 ++++++ .../v1/operator_configuration_type.go | 15 +++--- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 25 +++++---- pkg/cluster/k8sres.go | 4 +- pkg/cluster/sync_test.go | 4 +- pkg/controller/operator_config.go | 12 ++--- pkg/util/config/config.go | 3 +- 15 files changed, 237 insertions(+), 86 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index a790125d1..4c5ebdf66 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -321,6 +321,24 @@ spec: connection_pool: type: object properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -336,25 +354,7 @@ spec: connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100Mi" - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" + #default: "100m" status: type: object additionalProperties: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 8b7de363c..aa4d40b1d 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -242,6 +242,22 @@ spec: type: string replicaLoadBalancer: # deprecated type: boolean + connectionPool: + type: object + properties: + schema: + type: string + user: + type: string + replicas: + type: integer + dockerImage: + type: string + mode: + type: string + enum: + - "session" + - "transaction" resources: type: object required: diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 17b62226a..cc16a0979 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -262,15 +262,21 @@ configScalyr: scalyr_memory_request: 50Mi configConnectionPool: - connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "1" - connection_pool_default_memory_limit: 100m - connection_pool_default_memory_request: "100Mi" - # connection_pool_image: "" - connection_pool_instances_number: 1 - connection_pool_mode: "transaction" - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + # number of pooler instances + connection_pool_replicas: 1 + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # default pooling mode + connection_pool_mode: "transaction" + # default resources + connection_pool_default_cpu_request: "100m" + connection_pool_default_memory_request: "100M" + connection_pool_default_cpu_limit: "100m" + connection_pool_default_memory_limit: "100M" rbac: # Specifies whether RBAC resources should be created diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index f11619c8a..1f4cb6f70 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -239,15 +239,21 @@ configTeamsApi: # configure connection pooler deployment created by the operator configConnectionPool: - connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "1" - connection_pool_default_memory_limit: 100m - connection_pool_default_memory_request: "100Mi" - # connection_pool_image: "" - connection_pool_instances_number: 1 + # number of pooler instances + connection_pool_replicas: 1 + # db schema to install lookup function into + connection_pool_schema: "pooler" + # db user for pooler to use + connection_pool_user: "pooler" + # docker image + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # default pooling mode connection_pool_mode: "transaction" - # connection_pool_schema: "pooler" - # connection_pool_user: "pooler" + # default resources + connection_pool_default_cpu_request: "100m" + connection_pool_default_memory_request: "100M" + connection_pool_default_cpu_limit: "100m" + connection_pool_default_memory_limit: "100M" rbac: # Specifies whether RBAC resources should be created diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 7b049b6fa..a49890d13 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -149,6 +149,11 @@ These parameters are grouped directly under the `spec` key in the manifest. [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Optional. Default is: "30 00 \* \* \*" +* enableConnectionPool + Tells the operator to create a connection pool with a database. If this + field is true, a connection pool deployment will be created even if + `connectionPool` section is empty. + ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is @@ -359,3 +364,25 @@ CPU and memory limits for the sidecar container. * **memory** memory limits for the sidecar container. Optional, overrides the `default_memory_limits` operator configuration parameter. Optional. + +## Connection pool + +Parameters are grouped under the `connectionPool` top-level key and specify +configuration for connection pool. If this section is not empty, a connection +pool will be created for a database even if `enableConnectionPool` is not +present. + +* **replicas** + How many instances of connection pool to create. + +* **mode** + In which mode to run connection pool, transaction or section. + +* **schema** + Schema to create for credentials lookup function. + +* **user** + User to create for connection pool to be able to connect to a database. + +* **resources** + Resource configuration for connection pool deployment. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index e3893ea31..52d4e66c1 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -592,3 +592,31 @@ scalyr sidecar. In the CRD-based configuration they are grouped under the * **scalyr_memory_limit** Memory limit value for the Scalyr sidecar. The default is `500Mi`. + +## Connection pool configuration + +Parameters are grouped under the `connection_pool` top-level key and specify +default configuration for connection pool, if a postgres manifest requests it +but do not specify some of the parameters. All of them are optional with the +operator being able to provide some reasonable defaults. + +* **connection_pool_replicas** + How many instances of connection pool to create. + +* **connection_pool_schema** + Schema to create for credentials lookup function. + +* **connection_pool_user** + User to create for connection pool to be able to connect to a database. + +* **connection_pool_image** + Docker image to use for connection pool deployment. + +* **connection_pool_mode** + Default pool mode, sesssion or transaction. + +* **connection_pool_default_cpu_request** + **connection_pool_default_memory_reques** + **connection_pool_default_cpu_limit** + **connection_pool_default_memory_limit** + Default resource configuration for connection pool deployment. diff --git a/docs/user.md b/docs/user.md index f81e11ede..47be76773 100644 --- a/docs/user.md +++ b/docs/user.md @@ -454,3 +454,56 @@ monitoring is outside the scope of operator responsibilities. See [configuration reference](reference/cluster_manifest.md) and [administrator documentation](administrator.md) for details on how backups are executed. + +## Connection pool + +The operator can create a database side connection pool for those applications, +where an application side pool is not feasible, but a number of connections is +high. To create a connection pool together with a database, modify the +manifest: + +```yaml +spec: + enableConnectionPool: true +``` + +This will tell the operator to create a connection pool with default +configuration, though which one can access the master via a separate service +`{cluster-name}-pooler`. In most of the cases provided default configuration +should be good enough. + +To configure a new connection pool, specify: + +``` +spec: + connectionPool: + # how many instances of connection pool to create + replicas: 1 + + # in which mode to run, session or transaction + mode: "transaction" + + # schema, which operator will create to install credentials lookup + # function + schema: "pooler" + + # user, which operator will create for connection pool + user: "pooler" + + # resources for each instance + resources: + requests: + cpu: "100m" + memory: "100M" + limits: + cpu: "100m" + memory: "100M" +``` + +By default `pgbouncer` is used to create a connection pool. To find out about +pool modes see [docs](https://www.pgbouncer.org/config.html#pool_mode) (but it +should be general approach between different implementation). + +Note, that using `pgbouncer` means meaningful resource CPU limit should be less +than 1 core (there is a way to utilize more than one, but in K8S it's easier +just to spin up more instances). diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index f4224244d..aa64c6f6d 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -297,6 +297,24 @@ spec: connection_pool: type: object properties: + connection_pool_schema: + type: string + #default: "pooler" + connection_pool_user: + type: string + #default: "pooler" + connection_pool_instances_number: + type: integer + #default: 1 + connection_pool_image: + type: string + #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0" + connection_pool_mode: + type: string + enum: + - "session" + - "transaction" + #default: "transaction" connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' @@ -313,24 +331,6 @@ spec: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' #default: "100Mi" - connection_pool_image: - type: string - #default: "pierone.stups.zalan.do/acid/pgbouncer:0.0.1" - connection_pool_instances_number: - type: integer - #default: 1 - connection_pool_mode: - type: string - enum: - - "session" - - "transaction" - #default: "transaction" - connection_pool_schema: - type: string - #default: "pooler" - connection_pool_user: - type: string - #default: "pooler" status: type: object additionalProperties: diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index 7d4bb228b..ff9366421 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -206,6 +206,22 @@ spec: type: string replicaLoadBalancer: # deprecated type: boolean + connectionPool: + type: object + properties: + schema: + type: string + user: + type: string + replicas: + type: integer + dockerImage: + type: string + mode: + type: string + enum: + - "session" + - "transaction" resources: type: object required: diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 58f171843..dd0822c6a 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -65,12 +65,12 @@ type KubernetesMetaConfiguration struct { // TODO: use a proper toleration structure? PodToleration map[string]string `json:"toleration,omitempty"` // TODO: use namespacedname - PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` - PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` - MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` - EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` - PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` - PodManagementPolicy string `json:"pod_management_policy,omitempty"` + PodEnvironmentConfigMap string `json:"pod_environment_configmap,omitempty"` + PodPriorityClassName string `json:"pod_priority_class_name,omitempty"` + MasterPodMoveTimeout Duration `json:"master_pod_move_timeout,omitempty"` + EnablePodAntiAffinity bool `json:"enable_pod_antiaffinity,omitempty"` + PodAntiAffinityTopologyKey string `json:"pod_antiaffinity_topology_key,omitempty"` + PodManagementPolicy string `json:"pod_management_policy,omitempty"` } // PostgresPodResourcesDefaults defines the spec of default resources @@ -154,10 +154,9 @@ type ScalyrConfiguration struct { // Defines default configuration for connection pool type ConnectionPoolConfiguration struct { - NumberOfInstances *int32 `json:"connection_pool_instances_number,omitempty"` + Replicas *int32 `json:"connection_pool_replicas,omitempty"` Schema string `json:"connection_pool_schema,omitempty"` User string `json:"connection_pool_user,omitempty"` - Type string `json:"connection_pool_type,omitempty"` Image string `json:"connection_pool_image,omitempty"` Mode string `json:"connection_pool_mode,omitempty"` DefaultCPURequest string `name:"connection_pool_default_cpu_request,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index c56d70626..e7965f893 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -27,7 +27,7 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - EnableConnectionPool bool `json:"enable_connection_pool,omitempty"` + EnableConnectionPool bool `json:"enableConnectionPool,omitempty"` ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` TeamID string `json:"teamId"` @@ -159,16 +159,21 @@ type PostgresStatus struct { } // Options for connection pooler +// +// TODO: prepared snippets of configuration, one can choose via type, e.g. +// pgbouncer-large (with higher resources) or odyssey-small (with smaller +// resources) +// Type string `json:"type,omitempty"` +// +// TODO: figure out what other important parameters of the connection pool it +// makes sense to expose. E.g. pool size (min/max boundaries), max client +// connections etc. type ConnectionPool struct { - NumberOfInstances *int32 `json:"instancesNumber,omitempty"` - Schema string `json:"schema,omitempty"` - User string `json:"user,omitempty"` - Mode string `json:"mode,omitempty"` - DockerImage string `json:"dockerImage,omitempty"` - // TODO: prepared snippets of configuration, one can choose via type, e.g. - // pgbouncer-large (with higher resources) or odyssey-small (with smaller - // resources) - // Type string `json:"type,omitempty"` + Replicas *int32 `json:"replicas,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 3121691d9..b9f8e1992 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1861,9 +1861,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { podTemplate, err := c.generateConnPoolPodTemplate(spec) - numberOfInstances := spec.ConnectionPool.NumberOfInstances + numberOfInstances := spec.ConnectionPool.Replicas if numberOfInstances == nil { - numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances + numberOfInstances = c.OpConfig.ConnectionPool.Replicas } if err != nil { diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index c6928a64e..f5887dede 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -99,14 +99,14 @@ func TestConnPoolSynchronization(t *testing.T) { oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{ - NumberOfInstances: int32ToPointer(1), + Replicas: int32ToPointer(1), }, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{ - NumberOfInstances: int32ToPointer(2), + Replicas: int32ToPointer(2), }, }, }, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 1748fbd1f..a4a32abba 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -146,13 +146,13 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // Connection pool. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. - result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances - if result.ConnectionPool.NumberOfInstances == nil || - *result.ConnectionPool.NumberOfInstances < 1 { + result.ConnectionPool.Replicas = fromCRD.ConnectionPool.Replicas + if result.ConnectionPool.Replicas == nil || + *result.ConnectionPool.Replicas < 1 { var value int32 value = 1 - result.ConnectionPool.NumberOfInstances = &value + result.ConnectionPool.Replicas = &value } result.ConnectionPool.Schema = util.Coalesce( @@ -163,10 +163,6 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur fromCRD.ConnectionPool.User, constants.ConnectionPoolUserName) - result.ConnectionPool.Type = util.Coalesce( - fromCRD.ConnectionPool.Type, - constants.ConnectionPoolDefaultType) - result.ConnectionPool.Image = util.Coalesce( fromCRD.ConnectionPool.Image, "pgbouncer:0.0.1") diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 2baf99931..b04206d53 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,10 +85,9 @@ type LogicalBackup struct { // Operator options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_instances_number" default:"1"` + Replicas *int32 `name:"connection_pool_replicas" default:"1"` Schema string `name:"connection_pool_schema" default:"pooler"` User string `name:"connection_pool_user" default:"pooler"` - Type string `name:"connection_pool_type" default:"pgbouncer"` Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` Mode string `name:"connection_pool_mode" default:"session"` ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"100m"` From 4f2457bbe8f4a776156183027aa83ac5183c2d5e Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 14:19:15 +0100 Subject: [PATCH 19/61] clean up merge conflict --- manifests/operatorconfiguration.crd.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 463bdc2a6..1c4443ba3 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -303,11 +303,7 @@ spec: connection_pool_user: type: string #default: "pooler" -<<<<<<< HEAD connection_pool_replicas: -======= - connection_pool_instances_number: ->>>>>>> 2384e1ec10638e1989c0295ffb7fbe96af2e2f1a type: integer #default: 1 connection_pool_image: From 6ddac2f9d6ff54acdee2fdb0ce642ad7012c85bd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 13 Feb 2020 14:51:43 +0100 Subject: [PATCH 20/61] update codegen --- .../acid.zalan.do/v1/zz_generated.deepcopy.go | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index aaae1f04b..bc8e0d711 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -68,6 +68,49 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + out.Resources = in.Resources + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPool. +func (in *ConnectionPool) DeepCopy() *ConnectionPool { + if in == nil { + return nil + } + out := new(ConnectionPool) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) { + *out = *in + if in.Replicas != nil { + in, out := &in.Replicas, &out.Replicas + *out = new(int32) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ConnectionPoolConfiguration. +func (in *ConnectionPoolConfiguration) DeepCopy() *ConnectionPoolConfiguration { + if in == nil { + return nil + } + out := new(ConnectionPoolConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesMetaConfiguration) DeepCopyInto(out *KubernetesMetaConfiguration) { *out = *in @@ -254,6 +297,7 @@ func (in *OperatorConfigurationData) DeepCopyInto(out *OperatorConfigurationData out.LoggingRESTAPI = in.LoggingRESTAPI out.Scalyr = in.Scalyr out.LogicalBackup = in.LogicalBackup + in.ConnectionPool.DeepCopyInto(&out.ConnectionPool) return } @@ -416,6 +460,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { out.Volume = in.Volume in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources + if in.ConnectionPool != nil { + in, out := &in.ConnectionPool, &out.ConnectionPool + *out = new(ConnectionPool) + (*in).DeepCopyInto(*out) + } if in.SpiloFSGroup != nil { in, out := &in.SpiloFSGroup, &out.SpiloFSGroup *out = new(int64) From e2664843ba86ea368c638dae1d1f09e9c9dada8e Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 15:22:40 +0100 Subject: [PATCH 21/61] Add a real pgbouncer image --- manifests/postgresql-operator-default-configuration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index e9ce07fda..5a190e2ab 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -126,7 +126,7 @@ configuration: connection_pool_default_cpu_request: "1" connection_pool_default_memory_limit: 100m connection_pool_default_memory_request: "100Mi" - # connection_pool_image: "" + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-3" connection_pool_replicas: 1 connection_pool_mode: "transaction" # connection_pool_schema: "pooler" From e11f7876645c8e74786e8f2937d4e9b1aa2cd64c Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 16:34:02 +0100 Subject: [PATCH 22/61] Rename replicas to avoid potential confusion about terminology --- .../postgres-operator/crds/operatorconfigurations.yaml | 2 +- charts/postgres-operator/crds/postgresqls.yaml | 2 +- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- docs/reference/cluster_manifest.md | 2 +- docs/reference/operator_parameters.md | 2 +- docs/user.md | 2 +- manifests/configmap.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 2 +- .../postgresql-operator-default-configuration.yaml | 2 +- manifests/postgresql.crd.yaml | 2 +- .../acid.zalan.do/v1/operator_configuration_type.go | 2 +- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 10 +++++----- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 8 ++++---- pkg/cluster/k8sres.go | 4 ++-- pkg/cluster/sync_test.go | 4 ++-- pkg/controller/operator_config.go | 8 ++++---- pkg/util/config/config.go | 2 +- 18 files changed, 30 insertions(+), 30 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index c7325907a..5979cdf72 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -327,7 +327,7 @@ spec: connection_pool_user: type: string #default: "pooler" - connection_pool_replicas: + connection_pool_number_of_instances: type: integer #default: 1 connection_pool_image: diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 1efdbc7b7..5ba2a3a37 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -251,7 +251,7 @@ spec: type: string user: type: string - replicas: + number_of_instances: type: integer dockerImage: type: string diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index cc16a0979..bb40011ed 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -263,7 +263,7 @@ configScalyr: configConnectionPool: # number of pooler instances - connection_pool_replicas: 1 + connection_pool_number_of_instances: 1 # db schema to install lookup function into connection_pool_schema: "pooler" # db user for pooler to use diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 1f4cb6f70..1a028addd 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -240,7 +240,7 @@ configTeamsApi: # configure connection pooler deployment created by the operator configConnectionPool: # number of pooler instances - connection_pool_replicas: 1 + connection_pool_number_of_instances: 1 # db schema to install lookup function into connection_pool_schema: "pooler" # db user for pooler to use diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index a49890d13..da4756ca6 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -372,7 +372,7 @@ configuration for connection pool. If this section is not empty, a connection pool will be created for a database even if `enableConnectionPool` is not present. -* **replicas** +* **number_of_instances** How many instances of connection pool to create. * **mode** diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 52d4e66c1..665a4c992 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -600,7 +600,7 @@ default configuration for connection pool, if a postgres manifest requests it but do not specify some of the parameters. All of them are optional with the operator being able to provide some reasonable defaults. -* **connection_pool_replicas** +* **connection_pool_number_of_instances** How many instances of connection pool to create. * **connection_pool_schema** diff --git a/docs/user.md b/docs/user.md index 7758d1c34..64e5b98d1 100644 --- a/docs/user.md +++ b/docs/user.md @@ -478,7 +478,7 @@ To configure a new connection pool, specify: spec: connectionPool: # how many instances of connection pool to create - replicas: 1 + number_of_instances: 1 # in which mode to run, session or transaction mode: "transaction" diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6d02f0f58..d94e0ec55 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -16,7 +16,7 @@ data: # connection_pool_default_memory_limit: 100m # connection_pool_default_memory_request: "100Mi" # connection_pool_image: "" - # connection_pool_replicas: 1 + # connection_pool_number_of_instances: 1 # connection_pool_mode: "transaction" # connection_pool_schema: "pooler" # connection_pool_user: "pooler" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 1c4443ba3..dd303ea7e 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -303,7 +303,7 @@ spec: connection_pool_user: type: string #default: "pooler" - connection_pool_replicas: + connection_pool_number_of_instances: type: integer #default: 1 connection_pool_image: diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 5a190e2ab..d627767cd 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -127,7 +127,7 @@ configuration: connection_pool_default_memory_limit: 100m connection_pool_default_memory_request: "100Mi" connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-3" - connection_pool_replicas: 1 + connection_pool_number_of_instances: 1 connection_pool_mode: "transaction" # connection_pool_schema: "pooler" # connection_pool_user: "pooler" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index a85c81043..caee35ab0 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -215,7 +215,7 @@ spec: type: string user: type: string - replicas: + number_of_instances: type: integer dockerImage: type: string diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index dd0822c6a..bfdf61eda 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -154,7 +154,7 @@ type ScalyrConfiguration struct { // Defines default configuration for connection pool type ConnectionPoolConfiguration struct { - Replicas *int32 `json:"connection_pool_replicas,omitempty"` + NumberOfInstances *int32 `json:"connection_pool_number_of_instances,omitempty"` Schema string `json:"connection_pool_schema,omitempty"` User string `json:"connection_pool_user,omitempty"` Image string `json:"connection_pool_image,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index e7965f893..2679f51e1 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -169,11 +169,11 @@ type PostgresStatus struct { // makes sense to expose. E.g. pool size (min/max boundaries), max client // connections etc. type ConnectionPool struct { - Replicas *int32 `json:"replicas,omitempty"` - Schema string `json:"schema,omitempty"` - User string `json:"user,omitempty"` - Mode string `json:"mode,omitempty"` - DockerImage string `json:"dockerImage,omitempty"` + NumberOfInstances *int32 `json:"number_of_instances,omitempty"` + Schema string `json:"schema,omitempty"` + User string `json:"user,omitempty"` + Mode string `json:"mode,omitempty"` + DockerImage string `json:"dockerImage,omitempty"` Resources `json:"resources,omitempty"` } diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index bc8e0d711..a71e17d1f 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -71,8 +71,8 @@ func (in *CloneDescription) DeepCopy() *CloneDescription { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas + if in.NumberOfInstances != nil { + in, out := &in.NumberOfInstances, &out.NumberOfInstances *out = new(int32) **out = **in } @@ -93,8 +93,8 @@ func (in *ConnectionPool) DeepCopy() *ConnectionPool { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfiguration) { *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas + if in.NumberOfInstances != nil { + in, out := &in.NumberOfInstances, &out.NumberOfInstances *out = new(int32) **out = **in } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index b9f8e1992..3121691d9 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1861,9 +1861,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { podTemplate, err := c.generateConnPoolPodTemplate(spec) - numberOfInstances := spec.ConnectionPool.Replicas + numberOfInstances := spec.ConnectionPool.NumberOfInstances if numberOfInstances == nil { - numberOfInstances = c.OpConfig.ConnectionPool.Replicas + numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances } if err != nil { diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index f5887dede..c6928a64e 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -99,14 +99,14 @@ func TestConnPoolSynchronization(t *testing.T) { oldSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{ - Replicas: int32ToPointer(1), + NumberOfInstances: int32ToPointer(1), }, }, }, newSpec: &acidv1.Postgresql{ Spec: acidv1.PostgresSpec{ ConnectionPool: &acidv1.ConnectionPool{ - Replicas: int32ToPointer(2), + NumberOfInstances: int32ToPointer(2), }, }, }, diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index a4a32abba..ecec0307f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -146,13 +146,13 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // Connection pool. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. - result.ConnectionPool.Replicas = fromCRD.ConnectionPool.Replicas - if result.ConnectionPool.Replicas == nil || - *result.ConnectionPool.Replicas < 1 { + result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances + if result.ConnectionPool.NumberOfInstances == nil || + *result.ConnectionPool.NumberOfInstances < 1 { var value int32 value = 1 - result.ConnectionPool.Replicas = &value + result.ConnectionPool.NumberOfInstances = &value } result.ConnectionPool.Schema = util.Coalesce( diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index b04206d53..91c1acc0a 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -85,7 +85,7 @@ type LogicalBackup struct { // Operator options for connection pooler type ConnectionPool struct { - Replicas *int32 `name:"connection_pool_replicas" default:"1"` + NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"1"` Schema string `name:"connection_pool_schema" default:"pooler"` User string `name:"connection_pool_user" default:"pooler"` Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` From 8a81bc7464aa2983f6710dc6d2d26f87184216b1 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 17:03:59 +0100 Subject: [PATCH 23/61] Capture output for debugging purposes --- e2e/tests/test_e2e.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index e92aba11f..334de736f 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -423,7 +423,10 @@ def update_config(self, config_map_patch): self.wait_for_operator_pod_start() def create_with_kubectl(self, path): - subprocess.run(["kubectl", "create", "-f", path]) + subprocess.run( + ["kubectl", "create", "-f", path], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) if __name__ == '__main__': From 898d441ff2f8d965a2969c3758bb6d8cab7fbe86 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 13 Feb 2020 17:26:24 +0100 Subject: [PATCH 24/61] And output the resulting completed process --- e2e/tests/test_e2e.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 334de736f..21b083739 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -47,7 +47,8 @@ def setUpClass(cls): for filename in ["operator-service-account-rbac.yaml", "configmap.yaml", "postgres-operator.yaml"]: - k8s.create_with_kubectl("manifests/" + filename) + result = k8s.create_with_kubectl("manifests/" + filename) + print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) k8s.wait_for_operator_pod_start() @@ -55,7 +56,8 @@ def setUpClass(cls): 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish - k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") + print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) k8s.wait_for_pod_start('spilo-role=master') @timeout_decorator.timeout(TEST_TIMEOUT_SEC) @@ -423,7 +425,7 @@ def update_config(self, config_map_patch): self.wait_for_operator_pod_start() def create_with_kubectl(self, path): - subprocess.run( + return subprocess.run( ["kubectl", "create", "-f", path], stdout=subprocess.PIPE, stderr=subprocess.PIPE) From 447a659d6df9cbb7bd42ba97deb3c3438dd6ca06 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 14 Feb 2020 11:27:05 +0100 Subject: [PATCH 25/61] Report status in e2e --- e2e/tests/test_e2e.py | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 21b083739..7f526c576 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -56,9 +56,13 @@ def setUpClass(cls): 'default', label_selector='name=postgres-operator').items[0].spec.containers[0].image print("Tested operator image: {}".format(actual_operator_image)) # shows up after tests finish - result = k8s.create_with_kubectl("manifests/minimal-postgres-manifest.yaml") - print("stdout: {}, stderr: {}".format(result.stdout, result.stderr)) - k8s.wait_for_pod_start('spilo-role=master') + result = k8s.create_with_kubectl('manifests/minimal-postgres-manifest.yaml') + print('stdout: {}, stderr: {}'.format(result.stdout, result.stderr)) + try: + k8s.wait_for_pod_start('spilo-role=master') + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_min_resource_limits(self): @@ -356,12 +360,39 @@ def wait_for_operator_pod_start(self): # for local execution ~ 10 seconds suffices time.sleep(60) + def get_operator_pod(self): + pods = self.api.core_v1.list_namespaced_pod( + 'default', label_selector='name=postgres-operator' + ).items + + if pods: + return pods[0] + + return None + + def get_operator_log(self): + operator_pod = self.get_operator_pod() + pod_name = operator_pod.metadata.name + return self.api.core_v1.read_namespaced_pod_log( + name=pod_name, + namespace='default' + ) + def wait_for_pod_start(self, pod_labels, namespace='default'): pod_phase = 'No pod running' while pod_phase != 'Running': pods = self.api.core_v1.list_namespaced_pod(namespace, label_selector=pod_labels).items if pods: pod_phase = pods[0].status.phase + + if pods and pod_phase != 'Running': + pod_name = pods[0].metadata.name + response = self.api.core_v1.read_namespaced_pod( + name=pod_name, + namespace=namespace + ) + print("Pod description {}".format(response)) + time.sleep(self.RETRY_TIMEOUT_SEC) def check_service_annotations(self, svc_labels, annotations, namespace='default'): From 0095be0279f8bc0dbc180ce3a40288dfedfb4bb5 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 14 Feb 2020 13:04:24 +0100 Subject: [PATCH 26/61] Fix uninitialized ConnectionPool structure usage It's being used quite early to setup a connection pool user, and if only enableConnectionPool specified, can be nil. --- pkg/cluster/cluster.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 8e65b12ea..3f09e4e82 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -863,6 +863,11 @@ func (c *Cluster) initSystemUsers() { // created by operator as a normal pgUser if c.needConnectionPool() { + // initialize empty connection pool if not done yet + if c.Spec.ConnectionPool == nil { + c.Spec.ConnectionPool = &acidv1.ConnectionPool{} + } + username := c.Spec.ConnectionPool.User if username == "" { username = c.OpConfig.ConnectionPool.User From a9d02bacc4c7e67ea4cf08514d47e49f012865cb Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 14 Feb 2020 14:20:35 +0100 Subject: [PATCH 27/61] Address review --- charts/postgres-operator/values-crd.yaml | 4 ++-- charts/postgres-operator/values.yaml | 4 ++-- docs/reference/cluster_manifest.md | 5 ++++- manifests/postgresql.crd.yaml | 19 +++---------------- pkg/apis/acid.zalan.do/v1/crds.go | 8 ++++++++ .../v1/operator_configuration_type.go | 10 ++++++---- pkg/apis/acid.zalan.do/v1/postgresql_type.go | 6 ++++-- pkg/apis/acid.zalan.do/v1/register.go | 5 ++--- pkg/cluster/cluster.go | 4 ++-- pkg/cluster/k8sres_test.go | 12 ++++++------ pkg/cluster/resources.go | 4 ++-- pkg/cluster/resources_test.go | 14 +++++++++----- pkg/cluster/sync.go | 2 +- pkg/cluster/sync_test.go | 4 ++-- pkg/cluster/util.go | 6 +++++- pkg/util/constants/pooler.go | 6 +++--- 16 files changed, 61 insertions(+), 52 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index bb40011ed..13079e299 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -274,9 +274,9 @@ configConnectionPool: connection_pool_mode: "transaction" # default resources connection_pool_default_cpu_request: "100m" - connection_pool_default_memory_request: "100M" + connection_pool_default_memory_request: "100Mi" connection_pool_default_cpu_limit: "100m" - connection_pool_default_memory_limit: "100M" + connection_pool_default_memory_limit: "100Mi" rbac: # Specifies whether RBAC resources should be created diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 1a028addd..5d36e85ee 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -251,9 +251,9 @@ configConnectionPool: connection_pool_mode: "transaction" # default resources connection_pool_default_cpu_request: "100m" - connection_pool_default_memory_request: "100M" + connection_pool_default_memory_request: "100Mi" connection_pool_default_cpu_limit: "100m" - connection_pool_default_memory_limit: "100M" + connection_pool_default_memory_limit: "100Mi" rbac: # Specifies whether RBAC resources should be created diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index da4756ca6..4c5ce2f9c 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -372,7 +372,7 @@ configuration for connection pool. If this section is not empty, a connection pool will be created for a database even if `enableConnectionPool` is not present. -* **number_of_instances** +* **numberOfInstances** How many instances of connection pool to create. * **mode** @@ -384,5 +384,8 @@ present. * **user** User to create for connection pool to be able to connect to a database. +* **dockerImage** + Which docker image to use for connection pool deployment. + * **resources** Resource configuration for connection pool deployment. diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index caee35ab0..c809533a4 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -77,6 +77,9 @@ spec: type: string mode: type: string + enum: + - "session" + - "transaction" numberOfInstances: type: integer minimum: 1 @@ -208,22 +211,6 @@ spec: type: string replicaLoadBalancer: # deprecated type: boolean - connectionPool: - type: object - properties: - schema: - type: string - user: - type: string - number_of_instances: - type: integer - dockerImage: - type: string - mode: - type: string - enum: - - "session" - - "transaction" resources: type: object required: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 7cc7385f7..fe00189cc 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -184,6 +184,14 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "mode": { Type: "string", + Enum: []apiextv1beta1.JSON{ + { + Raw: []byte(`"session"`), + }, + { + Raw: []byte(`"transaction"`), + }, + }, }, "numberOfInstances": { Type: "integer", diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index bfdf61eda..0c4ff42b8 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -1,5 +1,7 @@ package v1 +// Operator configuration CRD definition, please use snake_case for field names. + import ( "github.com/zalando/postgres-operator/pkg/util/config" @@ -159,10 +161,10 @@ type ConnectionPoolConfiguration struct { User string `json:"connection_pool_user,omitempty"` Image string `json:"connection_pool_image,omitempty"` Mode string `json:"connection_pool_mode,omitempty"` - DefaultCPURequest string `name:"connection_pool_default_cpu_request,omitempty"` - DefaultMemoryRequest string `name:"connection_pool_default_memory_request,omitempty"` - DefaultCPULimit string `name:"connection_pool_default_cpu_limit,omitempty"` - DefaultMemoryLimit string `name:"connection_pool_default_memory_limit,omitempty"` + DefaultCPURequest string `json:"connection_pool_default_cpu_request,omitempty"` + DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"` + DefaultCPULimit string `json:"connection_pool_default_cpu_limit,omitempty"` + DefaultMemoryLimit string `json:"connection_pool_default_memory_limit,omitempty"` } // OperatorLogicalBackupConfiguration defines configuration for logical backup diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 2679f51e1..315154da2 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -1,5 +1,7 @@ package v1 +// Postgres CRD definition, please use CamelCase for field names. + import ( "time" @@ -27,7 +29,7 @@ type PostgresSpec struct { Patroni `json:"patroni,omitempty"` Resources `json:"resources,omitempty"` - EnableConnectionPool bool `json:"enableConnectionPool,omitempty"` + EnableConnectionPool *bool `json:"enableConnectionPool,omitempty"` ConnectionPool *ConnectionPool `json:"connectionPool,omitempty"` TeamID string `json:"teamId"` @@ -169,7 +171,7 @@ type PostgresStatus struct { // makes sense to expose. E.g. pool size (min/max boundaries), max client // connections etc. type ConnectionPool struct { - NumberOfInstances *int32 `json:"number_of_instances,omitempty"` + NumberOfInstances *int32 `json:"numberOfInstances,omitempty"` Schema string `json:"schema,omitempty"` User string `json:"user,omitempty"` Mode string `json:"mode,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/register.go b/pkg/apis/acid.zalan.do/v1/register.go index 34def209d..1c30e35fb 100644 --- a/pkg/apis/acid.zalan.do/v1/register.go +++ b/pkg/apis/acid.zalan.do/v1/register.go @@ -10,8 +10,7 @@ import ( // APIVersion of the `postgresql` and `operator` CRDs const ( - APIVersion = "v1" - PostgresqlKind = "postgresql" + APIVersion = "v1" ) var ( @@ -43,7 +42,7 @@ func addKnownTypes(scheme *runtime.Scheme) error { // AddKnownType assumes derives the type kind from the type name, which is always uppercase. // For our CRDs we use lowercase names historically, therefore we have to supply the name separately. // TODO: User uppercase CRDResourceKind of our types in the next major API version - scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind(PostgresqlKind), &Postgresql{}) + scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresql"), &Postgresql{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("postgresqlList"), &PostgresqlList{}) scheme.AddKnownTypeWithName(SchemeGroupVersion.WithKind("OperatorConfiguration"), &OperatorConfiguration{}) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 3f09e4e82..de70e1477 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -49,7 +49,7 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1beta1.RoleBinding } -type ConnectionPoolResources struct { +type ConnectionPoolObjects struct { Deployment *appsv1.Deployment Service *v1.Service } @@ -59,7 +59,7 @@ type kubeResources struct { Endpoints map[PostgresRole]*v1.Endpoints Secrets map[types.UID]*v1.Secret Statefulset *appsv1.StatefulSet - ConnectionPool *ConnectionPoolResources + ConnectionPool *ConnectionPoolObjects PodDisruptionBudget *policybeta1.PodDisruptionBudget //Pods are treated separately //PVCs are treated separately diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index c26e04b96..4b0a31628 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -541,8 +541,8 @@ func TestConnPoolPodSpec(t *testing.T) { ConnectionPool: config.ConnectionPool{ ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100M", - ConnPoolDefaultMemoryLimit: "100M", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -666,8 +666,8 @@ func TestConnPoolDeploymentSpec(t *testing.T) { ConnectionPool: config.ConnectionPool{ ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100M", - ConnPoolDefaultMemoryLimit: "100M", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) @@ -767,8 +767,8 @@ func TestConnPoolServiceSpec(t *testing.T) { ConnectionPool: config.ConnectionPool{ ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100M", - ConnPoolDefaultMemoryLimit: "100M", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index e44d50800..46be0af4b 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -97,7 +97,7 @@ func (c *Cluster) createStatefulSet() (*appsv1.StatefulSet, error) { // // After that create all the objects for connection pool, namely a deployment // with a chosen pooler and a service to expose it. -func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolResources, error) { +func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolObjects, error) { var msg string c.setProcessName("creating connection pool") @@ -144,7 +144,7 @@ func (c *Cluster) createConnectionPool(lookup InstallFunction) (*ConnectionPoolR return nil, err } - c.ConnectionPool = &ConnectionPoolResources{ + c.ConnectionPool = &ConnectionPoolObjects{ Deployment: deployment, Service: service, } diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index 7c52addad..d0cecf841 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -15,6 +15,10 @@ func mockInstallLookupFunction(schema string, user string) error { return nil } +func boolToPointer(value bool) *bool { + return &value +} + func TestConnPoolCreationAndDeletion(t *testing.T) { testName := "Test connection pool creation" var cluster = New( @@ -28,8 +32,8 @@ func TestConnPoolCreationAndDeletion(t *testing.T) { ConnectionPool: config.ConnectionPool{ ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100M", - ConnPoolDefaultMemoryLimit: "100M", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) @@ -77,8 +81,8 @@ func TestNeedConnPool(t *testing.T) { ConnectionPool: config.ConnectionPool{ ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100M", - ConnPoolDefaultMemoryLimit: "100M", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.NewMockKubernetesClient(), acidv1.Postgresql{}, logger) @@ -93,7 +97,7 @@ func TestNeedConnPool(t *testing.T) { } cluster.Spec = acidv1.PostgresSpec{ - EnableConnectionPool: true, + EnableConnectionPool: boolToPointer(true), } if !cluster.needConnectionPool() { diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index b59bf5533..f49638794 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -618,7 +618,7 @@ func (c *Cluster) syncLogicalBackupJob() error { func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error { if c.ConnectionPool == nil { c.logger.Warning("Connection pool resources are empty") - c.ConnectionPool = &ConnectionPoolResources{} + c.ConnectionPool = &ConnectionPoolObjects{} } deployment, err := c.KubeClient. diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index c6928a64e..4de75880c 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -54,8 +54,8 @@ func TestConnPoolSynchronization(t *testing.T) { ConnectionPool: config.ConnectionPool{ ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", - ConnPoolDefaultMemoryRequest: "100M", - ConnPoolDefaultMemoryLimit: "100M", + ConnPoolDefaultMemoryRequest: "100Mi", + ConnPoolDefaultMemoryLimit: "100Mi", }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index d2dd11586..3e53ffd45 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -497,5 +497,9 @@ func (c *Cluster) patroniUsesKubernetes() bool { } func (c *Cluster) needConnectionPool() bool { - return c.Spec.ConnectionPool != nil || c.Spec.EnableConnectionPool == true + if c.Spec.EnableConnectionPool == nil { + return c.Spec.ConnectionPool != nil + } else { + return *c.Spec.EnableConnectionPool + } } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index b25a12a6c..4d5edaa57 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -5,9 +5,9 @@ const ( ConnectionPoolUserName = "pooler" ConnectionPoolSchemaName = "pooler" ConnectionPoolDefaultType = "pgbouncer" - ConnectionPoolDefaultMode = "transition" + ConnectionPoolDefaultMode = "transaction" ConnectionPoolDefaultCpuRequest = "100m" ConnectionPoolDefaultCpuLimit = "100m" - ConnectionPoolDefaultMemoryRequest = "100M" - ConnectionPoolDefaultMemoryLimit = "100M" + ConnectionPoolDefaultMemoryRequest = "100Mi" + ConnectionPoolDefaultMemoryLimit = "100Mi" ) From 515bb2dfad2c0cb4cbf98a9bf54cc7723824e325 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 17 Feb 2020 13:07:40 +0100 Subject: [PATCH 28/61] Address review, add ConnectionPool init for sync --- docs/reference/cluster_manifest.md | 4 ++-- docs/reference/operator_parameters.md | 2 +- docs/user.md | 2 +- pkg/cluster/cluster.go | 18 +++++++++--------- pkg/cluster/k8sres.go | 21 +++++++++++++++++++++ pkg/cluster/resources.go | 2 +- 6 files changed, 35 insertions(+), 14 deletions(-) diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 4c5ce2f9c..a66cc169d 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -152,7 +152,7 @@ These parameters are grouped directly under the `spec` key in the manifest. * enableConnectionPool Tells the operator to create a connection pool with a database. If this field is true, a connection pool deployment will be created even if - `connectionPool` section is empty. + `connectionPool` section is empty. Optional, not set by default. ## Postgres parameters @@ -376,7 +376,7 @@ present. How many instances of connection pool to create. * **mode** - In which mode to run connection pool, transaction or section. + In which mode to run connection pool, transaction or session. * **schema** Schema to create for credentials lookup function. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 665a4c992..25ad46058 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -613,7 +613,7 @@ operator being able to provide some reasonable defaults. Docker image to use for connection pool deployment. * **connection_pool_mode** - Default pool mode, sesssion or transaction. + Default pool mode, session or transaction. * **connection_pool_default_cpu_request** **connection_pool_default_memory_reques** diff --git a/docs/user.md b/docs/user.md index 64e5b98d1..747b6dbe4 100644 --- a/docs/user.md +++ b/docs/user.md @@ -468,7 +468,7 @@ spec: ``` This will tell the operator to create a connection pool with default -configuration, though which one can access the master via a separate service +configuration, through which one can access the master via a separate service `{cluster-name}-pooler`. In most of the cases provided default configuration should be good enough. diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index de70e1477..94400066b 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1217,16 +1217,16 @@ func (c *Cluster) needSyncConnPoolDeployments(oldSpec, newSpec *acidv1.Connectio if err != nil { c.logger.Infof("Cannot get diff, do not do anything, %+v", err) return false, reasons - } else { - if len(changelog) > 0 { - sync = true - } + } - for _, change := range changelog { - msg := fmt.Sprintf("%s %+v from %s to %s", - change.Type, change.Path, change.From, change.To) - reasons = append(reasons, msg) - } + if len(changelog) > 0 { + sync = true + } + + for _, change := range changelog { + msg := fmt.Sprintf("%s %+v from %s to %s", + change.Type, change.Path, change.From, change.To) + reasons = append(reasons, msg) } return sync, reasons diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 3121691d9..9966f5b55 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1860,6 +1860,16 @@ func (c *Cluster) ownerReferences() []metav1.OwnerReference { func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( *appsv1.Deployment, error) { + // there are two ways to enable connection pooler, either to specify a + // connectionPool section or enableConnectionPool. In the second case + // spec.connectionPool will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPool == nil { + spec.ConnectionPool = &acidv1.ConnectionPool{} + } + podTemplate, err := c.generateConnPoolPodTemplate(spec) numberOfInstances := spec.ConnectionPool.NumberOfInstances if numberOfInstances == nil { @@ -1895,6 +1905,17 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( } func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service { + + // there are two ways to enable connection pooler, either to specify a + // connectionPool section or enableConnectionPool. In the second case + // spec.connectionPool will be nil, so to make it easier to calculate + // default values, initialize it to an empty structure. It could be done + // anywhere, but here is the earliest common entry point between sync and + // create code, so init here. + if spec.ConnectionPool == nil { + spec.ConnectionPool = &acidv1.ConnectionPool{} + } + serviceSpec := v1.ServiceSpec{ Ports: []v1.ServicePort{ { diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 46be0af4b..6b0648680 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -864,7 +864,7 @@ func (c *Cluster) updateConnPoolDeployment(oldDeploymentSpec, newDeployment *app } // An update probably requires RetryOnConflict, but since only one operator - // worker at one time will try to update it changes of conflicts are + // worker at one time will try to update it chances of conflicts are // minimal. deployment, err := c.KubeClient. Deployments(c.ConnectionPool.Deployment.Namespace). From 35d82e5a178d595954a494844f033f1d023d9a97 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 17 Feb 2020 14:28:38 +0100 Subject: [PATCH 29/61] Do sync also when there are no deployment --- pkg/cluster/sync.go | 40 +++++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 42c5e2250..47c8c01d6 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -110,17 +110,35 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } } - // connection pool - oldPool := oldSpec.Spec.ConnectionPool - newPool := newSpec.Spec.ConnectionPool - if c.needConnectionPool() && - (c.ConnectionPool == nil || !reflect.DeepEqual(oldPool, newPool)) { - - c.logger.Debug("syncing connection pool") - - if err := c.syncConnectionPool(&oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pool: %v", err) - return err + // sync connection pool + if c.needConnectionPool() { + oldPool := oldSpec.Spec.ConnectionPool + newPool := newSpec.Spec.ConnectionPool + + // do sync in case if any resources were not remembered (it means they + // probably were not created, or if specification differs + if c.ConnectionPool == nil || + c.ConnectionPool.Deployment == nil || + c.ConnectionPool.Service == nil || + !reflect.DeepEqual(oldPool, newPool) { + + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPool(&oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } + } else { + // check if we need to clean up connection pool resources after it was + // disabled + if c.ConnectionPool != nil && + (c.ConnectionPool.Deployment != nil || + c.ConnectionPool.Service != nil) { + + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } } } From 97217e27ed73aec0fe8ac74ed30ac96456d113d7 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 17 Feb 2020 15:34:10 +0100 Subject: [PATCH 30/61] Delete if a new specification is nil Use coalesce for username too. --- pkg/cluster/cluster.go | 7 +++---- pkg/cluster/sync.go | 8 ++++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 4f2fe4efb..1328646ac 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -868,10 +868,9 @@ func (c *Cluster) initSystemUsers() { c.Spec.ConnectionPool = &acidv1.ConnectionPool{} } - username := c.Spec.ConnectionPool.User - if username == "" { - username = c.OpConfig.ConnectionPool.User - } + username := util.Coalesce( + c.Spec.ConnectionPool.User, + c.OpConfig.ConnectionPool.User) c.systemUsers[constants.ConnectionPoolUserKeyName] = spec.PgUser{ Origin: spec.RoleConnectionPool, diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 47c8c01d6..bd4a7fc28 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -115,6 +115,14 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { oldPool := oldSpec.Spec.ConnectionPool newPool := newSpec.Spec.ConnectionPool + if newPool == nil { + // previously specified connectionPool was removed, so delete + // connection pool + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + // do sync in case if any resources were not remembered (it means they // probably were not created, or if specification differs if c.ConnectionPool == nil || From 66129335fd9974248f0f603b5e70c7be2a4ca553 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 19 Feb 2020 14:53:58 +0100 Subject: [PATCH 31/61] Adjust sync logic --- pkg/cluster/cluster.go | 18 +++---- pkg/cluster/resources.go | 2 +- pkg/cluster/sync.go | 108 ++++++++++++++++++++++++--------------- pkg/cluster/util.go | 12 +++-- 4 files changed, 83 insertions(+), 57 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 1328646ac..627c15481 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -600,7 +600,11 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - if !reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) { + // connection pool needs one system user created, which is done in + // initUsers. Check if it needs to be called. + sameUsers := reflect.DeepEqual(oldSpec.Spec.Users, newSpec.Spec.Users) + needConnPool := c.needConnectionPoolWorker(&newSpec.Spec) + if !sameUsers || needConnPool { c.logger.Debugf("syncing secrets") if err := c.initUsers(); err != nil { c.logger.Errorf("could not init users: %v", err) @@ -724,15 +728,9 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } } - // connection pool - if !reflect.DeepEqual(oldSpec.Spec.ConnectionPool, - newSpec.Spec.ConnectionPool) { - c.logger.Debug("syncing connection pool") - - if err := c.syncConnectionPool(oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pool: %v", err) - updateFailed = true - } + // sync connection pool + if err := c.syncConnectionPool(oldSpec, newSpec); err != nil { + return fmt.Errorf("could not sync connection pool: %v", err) } return nil diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index ecda3b44d..5c0a7bbd7 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -160,7 +160,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { // Lack of connection pooler objects is not a fatal error, just log it if // it was present before in the manifest - if c.needConnectionPool() && c.ConnectionPool == nil { + if c.ConnectionPool == nil { c.logger.Infof("No connection pool to delete") return nil } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index bd4a7fc28..f8d242d99 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -111,43 +111,8 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } // sync connection pool - if c.needConnectionPool() { - oldPool := oldSpec.Spec.ConnectionPool - newPool := newSpec.Spec.ConnectionPool - - if newPool == nil { - // previously specified connectionPool was removed, so delete - // connection pool - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) - } - } - - // do sync in case if any resources were not remembered (it means they - // probably were not created, or if specification differs - if c.ConnectionPool == nil || - c.ConnectionPool.Deployment == nil || - c.ConnectionPool.Service == nil || - !reflect.DeepEqual(oldPool, newPool) { - - c.logger.Debug("syncing connection pool") - - if err := c.syncConnectionPool(&oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pool: %v", err) - return err - } - } - } else { - // check if we need to clean up connection pool resources after it was - // disabled - if c.ConnectionPool != nil && - (c.ConnectionPool.Deployment != nil || - c.ConnectionPool.Service != nil) { - - if err := c.deleteConnectionPool(); err != nil { - c.logger.Warningf("could not remove connection pool: %v", err) - } - } + if err = c.syncConnectionPool(&oldSpec, newSpec); err != nil { + return fmt.Errorf("could not sync connection pool: %v", err) } return err @@ -499,10 +464,12 @@ func (c *Cluster) syncRoles() (err error) { userNames = append(userNames, u.Name) } - // An exception from system users, connection pool user - connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] - userNames = append(userNames, connPoolUser.Name) - c.pgUsers[connPoolUser.Name] = connPoolUser + if c.needConnectionPool() { + // An exception from system users, connection pool user + connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] + userNames = append(userNames, connPoolUser.Name) + c.pgUsers[connPoolUser.Name] = connPoolUser + } dbUsers, err = c.readPgUsersFromDatabase(userNames) if err != nil { @@ -637,11 +604,68 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } +func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error { + newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) + oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) + + if oldNeedConnPool && newNeedConnPool { + // sync in case of differences, or if no resources present + oldPool := oldSpec.Spec.ConnectionPool + newPool := newSpec.Spec.ConnectionPool + + if c.ConnectionPool == nil || + c.ConnectionPool.Deployment == nil || + c.ConnectionPool.Service == nil || + !reflect.DeepEqual(oldPool, newPool) { + + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } else { + c.logger.Debug("No connection pool sync") + } + } + + if !oldNeedConnPool && newNeedConnPool { + // sync to create everything + c.logger.Debug("syncing connection pool") + + if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { + c.logger.Errorf("could not sync connection pool: %v", err) + return err + } + } + + if oldNeedConnPool && !newNeedConnPool { + // delete and cleanup resources + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + + if !oldNeedConnPool && !newNeedConnPool { + // delete and cleanup resources if not empty + if c.ConnectionPool != nil && + (c.ConnectionPool.Deployment != nil || + c.ConnectionPool.Service != nil) { + + if err := c.deleteConnectionPool(); err != nil { + c.logger.Warningf("could not remove connection pool: %v", err) + } + } + } + + return nil +} + // Synchronize connection pool resources. Effectively we're interested only in // synchronizing the corresponding deployment, but in case of deployment or // service is missing, create it. After checking, also remember an object for // the future references. -func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error { +func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { if c.ConnectionPool == nil { c.logger.Warning("Connection pool resources are empty") c.ConnectionPool = &ConnectionPoolObjects{} diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 3e53ffd45..269e94b17 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -496,10 +496,14 @@ func (c *Cluster) patroniUsesKubernetes() bool { return c.OpConfig.EtcdHost == "" } -func (c *Cluster) needConnectionPool() bool { - if c.Spec.EnableConnectionPool == nil { - return c.Spec.ConnectionPool != nil +func (c *Cluster) needConnectionPoolWorker(spec *acidv1.PostgresSpec) bool { + if spec.EnableConnectionPool == nil { + return spec.ConnectionPool != nil } else { - return *c.Spec.EnableConnectionPool + return *spec.EnableConnectionPool } } + +func (c *Cluster) needConnectionPool() bool { + return c.needConnectionPoolWorker(&c.Spec) +} From 2afaa59719e5efcc3242518e50fbf4a3fbb15beb Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 19 Feb 2020 15:55:05 +0100 Subject: [PATCH 32/61] update code-gen --- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index a71e17d1f..b23fb3141 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -460,6 +460,11 @@ func (in *PostgresSpec) DeepCopyInto(out *PostgresSpec) { out.Volume = in.Volume in.Patroni.DeepCopyInto(&out.Patroni) out.Resources = in.Resources + if in.EnableConnectionPool != nil { + in, out := &in.EnableConnectionPool, &out.EnableConnectionPool + *out = new(bool) + **out = **in + } if in.ConnectionPool != nil { in, out := &in.ConnectionPool, &out.ConnectionPool *out = new(ConnectionPool) From 619c543b616b7243cdf80e5615a3e867b3299e14 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 20 Feb 2020 13:56:22 +0100 Subject: [PATCH 33/61] E2E tests for connection pool Turn off enableConnectionPool to be able to test it from the scratch. Add turn on, modify, turn off/on test. --- e2e/tests/test_e2e.py | 106 +++++++++++++++++++++++ manifests/configmap.yaml | 2 +- manifests/minimal-postgres-manifest.yaml | 1 - 3 files changed, 107 insertions(+), 2 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index e1f6fa1f9..92d6ce13e 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -9,6 +9,10 @@ from kubernetes import client, config +def to_selector(labels): + return ",".join(["=".join(l) for l in labels.items()]) + + class EndToEndTestCase(unittest.TestCase): ''' Test interaction of the operator with multiple K8s components. @@ -352,6 +356,91 @@ def test_service_annotations(self): } k8s.update_config(unpatch_custom_service_annotations) + @timeout_decorator.timeout(TEST_TIMEOUT_SEC) + def test_enable_disable_connection_pool(self): + ''' + For a database without connection pool, then turns it on, scale up, + turn off and on again. Test with different ways of doing this (via + enableConnectionPool or connectionPool configuration section). + ''' + k8s = self.k8s + service_labels = { + 'cluster-name': 'acid-minimal-cluster', + } + pod_labels = dict({ + 'connection-pool': 'acid-minimal-cluster-pooler', + }) + + pod_selector = to_selector(pod_labels) + service_selector = to_selector(service_labels) + + try: + # enable connection pool + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + + pods = k8s.api.core_v1.list_namespaced_pod( + 'default', label_selector=pod_selector + ).items + + self.assertTrue(pods, 'No connection pool pods') + + k8s.wait_for_service(service_selector) + services = k8s.api.core_v1.list_namespaced_service( + 'default', label_selector=service_selector + ).items + services = [ + s for s in services + if s.metadata.name.endswith('pooler') + ] + + self.assertTrue(services, 'No connection pool service') + + # scale up connection pool deployment + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'connectionPool': { + 'numberOfInstances': 2, + }, + } + }) + + k8s.wait_for_running_pods(pod_selector, 2) + + # turn it off, keeping configuration section + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': False, + } + }) + k8s.wait_for_pods_to_stop(pod_selector) + + k8s.api.custom_objects_api.patch_namespaced_custom_object( + 'acid.zalan.do', 'v1', 'default', + 'postgresqls', 'acid-minimal-cluster', + { + 'spec': { + 'enableConnectionPool': True, + } + }) + k8s.wait_for_pod_start(pod_selector) + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + raise + def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' Check that there is a single pod in the k8s cluster with the label "spilo-role=master" @@ -475,6 +564,23 @@ def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): while self.count_pods_with_label(labels) != number_of_instances: time.sleep(self.RETRY_TIMEOUT_SEC) + def wait_for_running_pods(self, labels, number, namespace=''): + while self.count_pods_with_label(labels) != number: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_pods_to_stop(self, labels, namespace=''): + while self.count_pods_with_label(labels) != 0: + time.sleep(self.RETRY_TIMEOUT_SEC) + + def wait_for_service(self, labels, namespace='default'): + def get_services(): + return self.api.core_v1.list_namespaced_service( + namespace, label_selector=labels + ).items + + while not get_services(): + time.sleep(self.RETRY_TIMEOUT_SEC) + def count_pods_with_label(self, labels, namespace='default'): return len(self.api.core_v1.list_namespaced_pod(namespace, label_selector=labels).items) diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index a68435b28..df15ee36e 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pool_default_cpu_request: "1" # connection_pool_default_memory_limit: 100m # connection_pool_default_memory_request: "100Mi" - # connection_pool_image: "" + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-3" # connection_pool_number_of_instances: 1 # connection_pool_mode: "transaction" # connection_pool_schema: "pooler" diff --git a/manifests/minimal-postgres-manifest.yaml b/manifests/minimal-postgres-manifest.yaml index 61b250c35..75dfdf07f 100644 --- a/manifests/minimal-postgres-manifest.yaml +++ b/manifests/minimal-postgres-manifest.yaml @@ -17,4 +17,3 @@ spec: foo: zalando # dbname: owner postgresql: version: "11" - enableConnectionPool: true From 3e9883270301457367c4476baf961160eeefadc3 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 24 Feb 2020 16:20:59 +0100 Subject: [PATCH 34/61] Add more tests sync logic system users init needConnectionPool --- pkg/cluster/cluster_test.go | 18 ++++++++++++ pkg/cluster/resources_test.go | 20 +++++++++++++ pkg/cluster/sync_test.go | 53 +++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) diff --git a/pkg/cluster/cluster_test.go b/pkg/cluster/cluster_test.go index 9efbc51c6..a1b361642 100644 --- a/pkg/cluster/cluster_test.go +++ b/pkg/cluster/cluster_test.go @@ -9,6 +9,7 @@ import ( acidv1 "github.com/zalando/postgres-operator/pkg/apis/acid.zalan.do/v1" "github.com/zalando/postgres-operator/pkg/spec" "github.com/zalando/postgres-operator/pkg/util/config" + "github.com/zalando/postgres-operator/pkg/util/constants" "github.com/zalando/postgres-operator/pkg/util/k8sutil" "github.com/zalando/postgres-operator/pkg/util/teams" v1 "k8s.io/api/core/v1" @@ -704,3 +705,20 @@ func TestServiceAnnotations(t *testing.T) { }) } } + +func TestInitSystemUsers(t *testing.T) { + testName := "Test system users initialization" + + // default cluster without connection pool + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; exist { + t.Errorf("%s, connection pool user is present", testName) + } + + // cluster with connection pool + cl.Spec.EnableConnectionPool = boolToPointer(true) + cl.initSystemUsers() + if _, exist := cl.systemUsers[constants.ConnectionPoolUserKeyName]; !exist { + t.Errorf("%s, connection pool user is not present", testName) + } +} diff --git a/pkg/cluster/resources_test.go b/pkg/cluster/resources_test.go index d0cecf841..f06e96e65 100644 --- a/pkg/cluster/resources_test.go +++ b/pkg/cluster/resources_test.go @@ -104,4 +104,24 @@ func TestNeedConnPool(t *testing.T) { t.Errorf("%s: Connection pool is not enabled with flag", testName) } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(false), + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is still enabled with flag being false", + testName) + } + + cluster.Spec = acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + ConnectionPool: &acidv1.ConnectionPool{}, + } + + if !cluster.needConnectionPool() { + t.Errorf("%s: Connection pool is not enabled with flag and full", + testName) + } } diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 4de75880c..ab58b2074 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -9,6 +9,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util/k8sutil" appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -41,6 +42,14 @@ func objectsAreSaved(cluster *Cluster, err error) error { return nil } +func objectsAreDeleted(cluster *Cluster, err error) error { + if cluster.ConnectionPool != nil { + return fmt.Errorf("Connection pool was not deleted") + } + + return nil +} + func TestConnPoolSynchronization(t *testing.T) { testName := "Test connection pool synchronization" var cluster = New( @@ -72,6 +81,13 @@ func TestConnPoolSynchronization(t *testing.T) { clusterMock := *cluster clusterMock.KubeClient = k8sutil.NewMockKubernetesClient() + clusterDirtyMock := *cluster + clusterDirtyMock.KubeClient = k8sutil.NewMockKubernetesClient() + clusterDirtyMock.ConnectionPool = &ConnectionPoolObjects{ + Deployment: &appsv1.Deployment{}, + Service: &v1.Service{}, + } + tests := []struct { subTest string oldSpec *acidv1.Postgresql @@ -94,6 +110,43 @@ func TestConnPoolSynchronization(t *testing.T) { cluster: &clusterMissingObjects, check: objectsAreSaved, }, + { + subTest: "create from scratch", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, + { + subTest: "delete if not needed", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: &clusterMock, + check: objectsAreDeleted, + }, + { + subTest: "cleanup if still there", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + cluster: &clusterDirtyMock, + check: objectsAreDeleted, + }, { subTest: "update deployment", oldSpec: &acidv1.Postgresql{ From 037d7120efc332558dec430f5fe78e64439816d0 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 2 Mar 2020 16:32:15 +0100 Subject: [PATCH 35/61] Sync due to defaults Since we can miss it while checking only spec --- pkg/cluster/cluster.go | 68 +++++++++++++++++++++++++++++++++++- pkg/cluster/k8sres.go | 2 +- pkg/cluster/sync.go | 6 ++-- pkg/cluster/sync_test.go | 20 +++++++++++ pkg/util/constants/pooler.go | 2 ++ pkg/util/k8sutil/k8sutil.go | 12 +++++++ 6 files changed, 106 insertions(+), 4 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 627c15481..6fe94d08a 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1206,7 +1206,7 @@ func (c *Cluster) deletePatroniClusterConfigMaps() error { // Test if two connection pool configuration needs to be synced. For simplicity // compare not the actual K8S objects, but the configuration itself and request // sync if there is any difference. -func (c *Cluster) needSyncConnPoolDeployments(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { +func (c *Cluster) needSyncConnPoolSpecs(oldSpec, newSpec *acidv1.ConnectionPool) (sync bool, reasons []string) { reasons = []string{} sync = false @@ -1228,3 +1228,69 @@ func (c *Cluster) needSyncConnPoolDeployments(oldSpec, newSpec *acidv1.Connectio return sync, reasons } + +func syncResources(a, b *v1.ResourceRequirements) bool { + for _, res := range []v1.ResourceName{ + v1.ResourceCPU, + v1.ResourceMemory, + } { + if !a.Limits[res].Equal(b.Limits[res]) || + !a.Requests[res].Equal(b.Requests[res]) { + return true + } + } + + return false +} + +// Check if we need to synchronize connection pool deploymend due to new +// defaults, that are different from what we see in the DeploymentSpec +func (c *Cluster) needSyncConnPoolDefaults( + spec *acidv1.ConnectionPool, + deployment *appsv1.Deployment) (sync bool, reasons []string) { + + reasons = []string{} + sync = false + + config := c.OpConfig.ConnectionPool + podTemplate := deployment.Spec.Template + poolContainer := podTemplate.Spec.Containers[constants.ConnPoolContainer] + + if spec.NumberOfInstances == nil && + deployment.Spec.Replicas != config.NumberOfInstances { + + sync = true + msg := fmt.Sprintf("NumberOfInstances is different (%d vs %d)", + deployment.Spec.Replicas, config.NumberOfInstances) + reasons = append(reasons, msg) + } + + if spec.DockerImage == "" && + poolContainer.Image != config.Image { + + sync = true + msg := fmt.Sprintf("DockerImage is different (%s vs %s)", + poolContainer.Image, config.Image) + reasons = append(reasons, msg) + } + + expectedResources, err := generateResourceRequirements(spec.Resources, + c.makeDefaultConnPoolResources()) + + // An error to generate expected resources means something is not quite + // right, but for the purpose of robustness do not panic here, just report + // and ignore resources comparison (in the worst case there will be no + // updates for new resource values). + if err == nil && syncResources(&poolContainer.Resources, expectedResources) { + sync = true + msg := fmt.Sprintf("Resources are different (%+v vs %+v)", + poolContainer.Resources, expectedResources) + reasons = append(reasons, msg) + } + + if err != nil { + c.logger.Warningf("Cannot generate expected resources, %v", err) + } + + return sync, reasons +} diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index b60204233..53c608e1d 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1754,7 +1754,7 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( secretSelector := func(key string) *v1.SecretKeySelector { return &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ - Name: c.credentialSecretName(c.OpConfig.SuperUsername), + Name: c.credentialSecretName(c.OpConfig.ConnectionPool.User), }, Key: key, } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index f8d242d99..307f703ee 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -702,8 +702,10 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) // actual synchronization oldConnPool := oldSpec.Spec.ConnectionPool newConnPool := newSpec.Spec.ConnectionPool - sync, reason := c.needSyncConnPoolDeployments(oldConnPool, newConnPool) - if sync { + specSync, specReason := c.needSyncConnPoolSpecs(oldConnPool, newConnPool) + defaultsSync, defaultsReason := c.needSyncConnPoolDefaults(newConnPool, deployment) + reason := append(specReason, defaultsReason...) + if specSync || defaultsSync { c.logger.Infof("Update connection pool deployment %s, reason: %s", c.connPoolName(), reason) diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index ab58b2074..8087a7db4 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -88,6 +88,11 @@ func TestConnPoolSynchronization(t *testing.T) { Service: &v1.Service{}, } + clusterNewDefaultsMock := *cluster + clusterNewDefaultsMock.KubeClient = k8sutil.NewMockKubernetesClient() + cluster.OpConfig.ConnectionPool.Image = "pooler:2.0" + cluster.OpConfig.ConnectionPool.NumberOfInstances = int32ToPointer(2) + tests := []struct { subTest string oldSpec *acidv1.Postgresql @@ -166,6 +171,21 @@ func TestConnPoolSynchronization(t *testing.T) { cluster: &clusterMock, check: deploymentUpdated, }, + { + subTest: "update image from changed defaults", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + ConnectionPool: &acidv1.ConnectionPool{}, + }, + }, + cluster: &clusterNewDefaultsMock, + check: deploymentUpdated, + }, } for _, tt := range tests { err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec) diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index 4d5edaa57..aa8171110 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -10,4 +10,6 @@ const ( ConnectionPoolDefaultCpuLimit = "100m" ConnectionPoolDefaultMemoryRequest = "100Mi" ConnectionPoolDefaultMemoryLimit = "100Mi" + + ConnPoolContainer = 0 ) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index dd062b209..6e0798312 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -300,6 +300,18 @@ func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1 ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: int32ToPointer(1), + Template: v1.PodTemplateSpec{ + Spec: v1.PodSpec{ + Containers: []v1.Container{ + v1.Container{ + Image: "pooler:1.0", + }, + }, + }, + }, + }, }, nil } From 918df1461b8c2d4589ea66fa17160a84cef6d66d Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Tue, 3 Mar 2020 15:58:33 +0100 Subject: [PATCH 36/61] Add possibility to set max db connections Since it's an important part of a connection pool configuration, allow to configure max db connections, that pool will open to a target db. From this numbers several others (like default pool size, min pool size, reserve) will be deduced, taking into account desired number of instances. --- .../v1/operator_configuration_type.go | 1 + pkg/apis/acid.zalan.do/v1/postgresql_type.go | 1 + pkg/cluster/k8sres.go | 95 ++++++++++++++++--- pkg/cluster/k8sres_test.go | 1 + pkg/controller/operator_config.go | 21 ++-- pkg/util/config/config.go | 1 + pkg/util/constants/pooler.go | 4 +- pkg/util/k8sutil/k8sutil.go | 6 +- pkg/util/util.go | 16 ++++ 9 files changed, 122 insertions(+), 24 deletions(-) diff --git a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go index 0c4ff42b8..4b2a46620 100644 --- a/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go +++ b/pkg/apis/acid.zalan.do/v1/operator_configuration_type.go @@ -161,6 +161,7 @@ type ConnectionPoolConfiguration struct { User string `json:"connection_pool_user,omitempty"` Image string `json:"connection_pool_image,omitempty"` Mode string `json:"connection_pool_mode,omitempty"` + MaxDBConnections *int32 `json:"connection_pool_max_db_connections,omitempty"` DefaultCPURequest string `json:"connection_pool_default_cpu_request,omitempty"` DefaultMemoryRequest string `json:"connection_pool_default_memory_request,omitempty"` DefaultCPULimit string `json:"connection_pool_default_cpu_limit,omitempty"` diff --git a/pkg/apis/acid.zalan.do/v1/postgresql_type.go b/pkg/apis/acid.zalan.do/v1/postgresql_type.go index 315154da2..771924be8 100644 --- a/pkg/apis/acid.zalan.do/v1/postgresql_type.go +++ b/pkg/apis/acid.zalan.do/v1/postgresql_type.go @@ -176,6 +176,7 @@ type ConnectionPool struct { User string `json:"user,omitempty"` Mode string `json:"mode,omitempty"` DockerImage string `json:"dockerImage,omitempty"` + MaxDBConnections *int32 `json:"maxDBConnections,omitempty"` Resources `json:"resources,omitempty"` } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 53c608e1d..67417f277 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -20,6 +20,7 @@ import ( "github.com/zalando/postgres-operator/pkg/util" "github.com/zalando/postgres-operator/pkg/util/config" "github.com/zalando/postgres-operator/pkg/util/constants" + "github.com/zalando/postgres-operator/pkg/util/k8sutil" batchv1 "k8s.io/api/batch/v1" batchv1beta1 "k8s.io/api/batch/v1beta1" "k8s.io/apimachinery/pkg/labels" @@ -1731,6 +1732,82 @@ func (c *Cluster) getLogicalBackupJobName() (jobName string) { return "logical-backup-" + c.clusterName().Name } +// Generate pool size related environment variables. +// +// MAX_DB_CONN would specify the global maximum for connections to a target +// database. +// +// MAX_CLIENT_CONN is not configurable at the moment, just set it high enough. +// +// DEFAULT_SIZE is a pool size per db/user (having in mind the use case when +// most of the queries coming through a connection pooler are from the same +// user to the same db). In case if we want to spin up more connection pool +// instances, take this into account and maintain the same number of +// connections. +// +// MIN_SIZE is a pool minimal size, to prevent situation when sudden workload +// have to wait for spinning up a new connections. +// +// RESERVE_SIZE is how many additional connections to allow for a pool. +func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { + effectiveMode := util.Coalesce( + spec.ConnectionPool.Mode, + c.OpConfig.ConnectionPool.Mode) + + numberOfInstances := spec.ConnectionPool.NumberOfInstances + if numberOfInstances == nil { + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPool.NumberOfInstances, + k8sutil.Int32ToPointer(1)) + } + + effectiveMaxDBConn := util.CoalesceInt32( + spec.ConnectionPool.MaxDBConnections, + c.OpConfig.ConnectionPool.MaxDBConnections) + + if effectiveMaxDBConn == nil { + effectiveMaxDBConn = k8sutil.Int32ToPointer( + constants.ConnPoolMaxDBConnections) + } + + maxDBConn := *effectiveMaxDBConn / *numberOfInstances + + defaultSize := maxDBConn / 2 + minSize := defaultSize / 2 + reserveSize := minSize + + return []v1.EnvVar{ + { + Name: "CONNECTION_POOL_PORT", + Value: fmt.Sprint(pgPort), + }, + { + Name: "CONNECTION_POOL_MODE", + Value: effectiveMode, + }, + { + Name: "CONNECTION_POOL_DEFAULT_SIZE", + Value: fmt.Sprint(defaultSize), + }, + { + Name: "CONNECTION_POOL_MIN_SIZE", + Value: fmt.Sprint(minSize), + }, + { + Name: "CONNECTION_POOL_RESERVE_SIZE", + Value: fmt.Sprint(reserveSize), + }, + { + Name: "CONNECTION_POOL_MAX_CLIENT_CONN", + Value: fmt.Sprint(constants.ConnPoolMaxClientConnections), + }, + { + Name: "CONNECTION_POOL_MAX_DB_CONN", + Value: fmt.Sprint(effectiveMaxDBConn), + }, + } +} + func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( *v1.PodTemplateSpec, error) { @@ -1739,10 +1816,6 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( spec.ConnectionPool.Resources, c.makeDefaultConnPoolResources()) - effectiveMode := util.Coalesce( - spec.ConnectionPool.Mode, - c.OpConfig.ConnectionPool.Mode) - effectiveDockerImage := util.Coalesce( spec.ConnectionPool.DockerImage, c.OpConfig.ConnectionPool.Image) @@ -1789,16 +1862,10 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( SecretKeyRef: secretSelector("password"), }, }, - { - Name: "CONNECTION_POOL_MODE", - Value: effectiveMode, - }, - { - Name: "CONNECTION_POOL_PORT", - Value: fmt.Sprint(pgPort), - }, } + envVars = append(envVars, c.getConnPoolEnvVars(spec)...) + poolerContainer := v1.Container{ Name: connectionPoolContainer, Image: effectiveDockerImage, @@ -1873,7 +1940,9 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( podTemplate, err := c.generateConnPoolPodTemplate(spec) numberOfInstances := spec.ConnectionPool.NumberOfInstances if numberOfInstances == nil { - numberOfInstances = c.OpConfig.ConnectionPool.NumberOfInstances + numberOfInstances = util.CoalesceInt32( + c.OpConfig.ConnectionPool.NumberOfInstances, + k8sutil.Int32ToPointer(1)) } if err != nil { diff --git a/pkg/cluster/k8sres_test.go b/pkg/cluster/k8sres_test.go index 4b0a31628..43b2eac8e 100644 --- a/pkg/cluster/k8sres_test.go +++ b/pkg/cluster/k8sres_test.go @@ -539,6 +539,7 @@ func TestConnPoolPodSpec(t *testing.T) { ReplicationUsername: replicationUserName, }, ConnectionPool: config.ConnectionPool{ + MaxDBConnections: int32ToPointer(60), ConnPoolDefaultCPURequest: "100m", ConnPoolDefaultCPULimit: "100m", ConnPoolDefaultMemoryRequest: "100Mi", diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index ecec0307f..e775b3336 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -22,6 +22,10 @@ func (c *Controller) readOperatorConfigurationFromCRD(configObjectNamespace, con return config, nil } +func int32ToPointer(value int32) *int32 { + return &value +} + // importConfigurationFromCRD is a transitional function that converts CRD configuration to the one based on the configmap func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigurationData) *config.Config { result := &config.Config{} @@ -146,14 +150,13 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // Connection pool. Looks like we can't use defaulting in CRD before 1.17, // so ensure default values here. - result.ConnectionPool.NumberOfInstances = fromCRD.ConnectionPool.NumberOfInstances - if result.ConnectionPool.NumberOfInstances == nil || - *result.ConnectionPool.NumberOfInstances < 1 { - var value int32 + result.ConnectionPool.NumberOfInstances = util.CoalesceInt32( + fromCRD.ConnectionPool.NumberOfInstances, + int32ToPointer(1)) - value = 1 - result.ConnectionPool.NumberOfInstances = &value - } + result.ConnectionPool.NumberOfInstances = util.MaxInt32( + result.ConnectionPool.NumberOfInstances, + int32ToPointer(1)) result.ConnectionPool.Schema = util.Coalesce( fromCRD.ConnectionPool.Schema, @@ -187,5 +190,9 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur fromCRD.ConnectionPool.DefaultMemoryLimit, constants.ConnectionPoolDefaultMemoryLimit) + result.ConnectionPool.MaxDBConnections = util.CoalesceInt32( + fromCRD.ConnectionPool.MaxDBConnections, + int32ToPointer(constants.ConnPoolMaxDBConnections)) + return result } diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 56455de9a..dae21a4ee 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -94,6 +94,7 @@ type ConnectionPool struct { ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"3"` ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"1Gi"` + MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` } // Config describes operator config diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index aa8171110..0426dbfe8 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -11,5 +11,7 @@ const ( ConnectionPoolDefaultMemoryRequest = "100Mi" ConnectionPoolDefaultMemoryLimit = "100Mi" - ConnPoolContainer = 0 + ConnPoolContainer = 0 + ConnPoolMaxDBConnections = 60 + ConnPoolMaxClientConnections = 10000 ) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 6e0798312..80d5df0d9 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -28,7 +28,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func int32ToPointer(value int32) *int32 { +func Int32ToPointer(value int32) *int32 { return &value } @@ -301,7 +301,7 @@ func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1 Name: "test-deployment", }, Spec: apiappsv1.DeploymentSpec{ - Replicas: int32ToPointer(1), + Replicas: Int32ToPointer(1), Template: v1.PodTemplateSpec{ Spec: v1.PodSpec{ Containers: []v1.Container{ @@ -318,7 +318,7 @@ func (mock *mockDeployment) Get(name string, opts metav1.GetOptions) (*apiappsv1 func (mock *mockDeployment) Patch(name string, t types.PatchType, data []byte, subres ...string) (*apiappsv1.Deployment, error) { return &apiappsv1.Deployment{ Spec: apiappsv1.DeploymentSpec{ - Replicas: int32ToPointer(2), + Replicas: Int32ToPointer(2), }, ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", diff --git a/pkg/util/util.go b/pkg/util/util.go index ad6de14a2..abb10d42e 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -141,6 +141,22 @@ func Coalesce(val, defaultVal string) string { return val } +// Yeah, golang +func CoalesceInt32(val, defaultVal *int32) *int32 { + if val == nil { + return defaultVal + } + return val +} + +func MaxInt32(a, b *int32) *int32 { + if *a > *b { + return a + } + + return b +} + // IsSmallerQuantity : checks if first resource is of a smaller quantity than the second func IsSmallerQuantity(requestStr, limitStr string) (bool, error) { From 2bba673f1d63e58c0e1eaa266da1832fa6bbb5fe Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 4 Mar 2020 09:36:42 +0100 Subject: [PATCH 37/61] update codegen --- pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go index b23fb3141..47174f079 100644 --- a/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go +++ b/pkg/apis/acid.zalan.do/v1/zz_generated.deepcopy.go @@ -76,6 +76,11 @@ func (in *ConnectionPool) DeepCopyInto(out *ConnectionPool) { *out = new(int32) **out = **in } + if in.MaxDBConnections != nil { + in, out := &in.MaxDBConnections, &out.MaxDBConnections + *out = new(int32) + **out = **in + } out.Resources = in.Resources return } @@ -98,6 +103,11 @@ func (in *ConnectionPoolConfiguration) DeepCopyInto(out *ConnectionPoolConfigura *out = new(int32) **out = **in } + if in.MaxDBConnections != nil { + in, out := &in.MaxDBConnections, &out.MaxDBConnections + *out = new(int32) + **out = **in + } return } From 4d12615c0d81b9ba142e32f39db80726140cbc7b Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Wed, 4 Mar 2020 10:38:01 +0100 Subject: [PATCH 38/61] reflect new pooler parameter in validation + define same pooler image everywhere --- .../crds/operatorconfigurations.yaml | 11 ++++++---- .../postgres-operator/crds/postgresqls.yaml | 21 +++++-------------- manifests/configmap.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 11 ++++++---- ...gresql-operator-default-configuration.yaml | 2 +- manifests/postgresql.crd.yaml | 2 ++ pkg/apis/acid.zalan.do/v1/crds.go | 12 ++++++++--- pkg/util/config/config.go | 6 +++--- 8 files changed, 35 insertions(+), 32 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 5979cdf72..a45607905 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -327,18 +327,21 @@ spec: connection_pool_user: type: string #default: "pooler" - connection_pool_number_of_instances: - type: integer - #default: 1 connection_pool_image: type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0" + #default: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + connection_pool_max_db_connections: + type: integer + #default: 60 connection_pool_mode: type: string enum: - "session" - "transaction" #default: "transaction" + connection_pool_number_of_instances: + type: integer + #default: 1 connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index 5ba2a3a37..89098d315 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -111,8 +111,13 @@ spec: properties: dockerImage: type: string + maxDBConnections: + type: integer mode: type: string + enum: + - "session" + - "transaction" numberOfInstances: type: integer minimum: 1 @@ -244,22 +249,6 @@ spec: type: string replicaLoadBalancer: # deprecated type: boolean - connectionPool: - type: object - properties: - schema: - type: string - user: - type: string - number_of_instances: - type: integer - dockerImage: - type: string - mode: - type: string - enum: - - "session" - - "transaction" resources: type: object required: diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index df15ee36e..f1c5446b0 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -15,7 +15,7 @@ data: # connection_pool_default_cpu_request: "1" # connection_pool_default_memory_limit: 100m # connection_pool_default_memory_request: "100Mi" - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-3" + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" # connection_pool_number_of_instances: 1 # connection_pool_mode: "transaction" # connection_pool_schema: "pooler" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index dd303ea7e..aa5db341d 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -303,18 +303,21 @@ spec: connection_pool_user: type: string #default: "pooler" - connection_pool_number_of_instances: - type: integer - #default: 1 connection_pool_image: type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer:1.0.0" + #default: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + connection_pool_max_db_connections: + type: integer + #default: 60 connection_pool_mode: type: string enum: - "session" - "transaction" #default: "transaction" + connection_pool_number_of_instances: + type: integer + #default: 1 connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 4f5c9d960..5bc20eae3 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -126,7 +126,7 @@ configuration: connection_pool_default_cpu_request: "1" connection_pool_default_memory_limit: 100m connection_pool_default_memory_request: "100Mi" - connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-3" + connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" connection_pool_number_of_instances: 1 connection_pool_mode: "transaction" # connection_pool_schema: "pooler" diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index c809533a4..4e938319b 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -75,6 +75,8 @@ spec: properties: dockerImage: type: string + maxDBConnections: + type: integer mode: type: string enum: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index fe00189cc..22bac54e0 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -182,6 +182,9 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ "dockerImage": { Type: "string", }, + "maxDBConnections": { + Type: "integer", + }, "mode": { Type: "string", Enum: []apiextv1beta1.JSON{ @@ -1129,9 +1132,8 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation "connection_pool_image": { Type: "string", }, - "connection_pool_replicas": { - Type: "integer", - Minimum: &min1, + "connection_pool_max_db_connections": { + Type: "integer", }, "connection_pool_mode": { Type: "string", @@ -1144,6 +1146,10 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, }, }, + "connection_pool_number_of_instances": { + Type: "integer", + Minimum: &min1, + }, "connection_pool_schema": { Type: "string", }, diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index dae21a4ee..8f6af5107 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -88,13 +88,13 @@ type ConnectionPool struct { NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"1"` Schema string `name:"connection_pool_schema" default:"pooler"` User string `name:"connection_pool_user" default:"pooler"` - Image string `name:"connection_pool_image" default:"pgbouncer:1.0"` - Mode string `name:"connection_pool_mode" default:"session"` + Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer:master-5"` + Mode string `name:"connection_pool_mode" default:"transaction"` + MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"100m"` ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"3"` ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"1Gi"` - MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` } // Config describes operator config From e0df9dea0c9abfd19e61a16227a3093dcd98ae33 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Wed, 4 Mar 2020 16:07:22 +0100 Subject: [PATCH 39/61] Sync in case of missing deployment If nothing changed we still need to try to sync and test if the deployment is there. Otherwise it could be deleted, and operator will not notice. --- pkg/cluster/sync.go | 30 ++++++------------------------ 1 file changed, 6 insertions(+), 24 deletions(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 307f703ee..4a38a7e77 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -2,7 +2,6 @@ package cluster import ( "fmt" - "reflect" batchv1beta1 "k8s.io/api/batch/v1beta1" v1 "k8s.io/api/core/v1" @@ -608,29 +607,12 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) - if oldNeedConnPool && newNeedConnPool { - // sync in case of differences, or if no resources present - oldPool := oldSpec.Spec.ConnectionPool - newPool := newSpec.Spec.ConnectionPool - - if c.ConnectionPool == nil || - c.ConnectionPool.Deployment == nil || - c.ConnectionPool.Service == nil || - !reflect.DeepEqual(oldPool, newPool) { - - c.logger.Debug("syncing connection pool") - - if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { - c.logger.Errorf("could not sync connection pool: %v", err) - return err - } - } else { - c.logger.Debug("No connection pool sync") - } - } - - if !oldNeedConnPool && newNeedConnPool { - // sync to create everything + if newNeedConnPool { + // Try to sync in any case. If we didn't needed connection pool before, + // it means we want to create it. If it was already present, still sync + // since it could happen that there is no difference in specs, and all + // the resources are remembered, but the deployment was manualy deleted + // in between c.logger.Debug("syncing connection pool") if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { From a38a5aa5744c0ce766d850892e3e602cb62cb786 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 5 Mar 2020 10:12:10 +0100 Subject: [PATCH 40/61] Add test for sync with flag --- pkg/cluster/sync_test.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 8087a7db4..33bd01d16 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -115,6 +115,19 @@ func TestConnPoolSynchronization(t *testing.T) { cluster: &clusterMissingObjects, check: objectsAreSaved, }, + { + subTest: "create if doesn't exist with a flag", + oldSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{}, + }, + newSpec: &acidv1.Postgresql{ + Spec: acidv1.PostgresSpec{ + EnableConnectionPool: boolToPointer(true), + }, + }, + cluster: &clusterMissingObjects, + check: objectsAreSaved, + }, { subTest: "create from scratch", oldSpec: &acidv1.Postgresql{ From 07adaf28cc8f570fa6e1a76d9602c455b766a028 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 5 Mar 2020 10:32:20 +0100 Subject: [PATCH 41/61] Extend labels for connection pool --- pkg/cluster/util.go | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 269e94b17..2f10a1868 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -414,11 +414,24 @@ func (c *Cluster) labelsSelector() *metav1.LabelSelector { } } +// Return connection pool labels selector, which should from one point of view +// inherit most of the labels from the cluster itself, but at the same time +// have e.g. different `application` label, so that recreatePod operation will +// not interfere with it (it lists all the pods via labels, and if there would +// be no difference, it will recreate also pooler pods). func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { + labels := c.labelsSet(false) + connPoolLabels := map[string]string{ + "connection-pool": c.connPoolName(), + "application": "connection-pool", + } + + for k, v := range connPoolLabels { + labels[k] = v + } + return &metav1.LabelSelector{ - MatchLabels: map[string]string{ - "connection-pool": c.connPoolName(), - }, + MatchLabels: labels, MatchExpressions: nil, } } From f1646c8204b8b7796ad3ed44304d05c5580e752f Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Thu, 5 Mar 2020 11:08:50 +0100 Subject: [PATCH 42/61] minor changes to docs and manifests --- charts/postgres-operator/values-crd.yaml | 6 ++++-- charts/postgres-operator/values.yaml | 6 ++++-- docs/reference/cluster_manifest.md | 19 +++++++++++-------- docs/reference/operator_parameters.md | 3 +++ manifests/configmap.yaml | 3 ++- ...gresql-operator-default-configuration.yaml | 3 ++- pkg/controller/operator_config.go | 2 +- 7 files changed, 27 insertions(+), 15 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 0232000f0..187e9df9a 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -264,16 +264,18 @@ configScalyr: scalyr_memory_request: 50Mi configConnectionPool: - # number of pooler instances - connection_pool_number_of_instances: 1 # db schema to install lookup function into connection_pool_schema: "pooler" # db user for pooler to use connection_pool_user: "pooler" # docker image connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # max db connections the pooler should hold + connection_pool_max_db_connections: 60 # default pooling mode connection_pool_mode: "transaction" + # number of pooler instances + connection_pool_number_of_instances: 1 # default resources connection_pool_default_cpu_request: "100m" connection_pool_default_memory_request: "100Mi" diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index e0b30f137..3a7c580fd 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -241,16 +241,18 @@ configTeamsApi: # configure connection pooler deployment created by the operator configConnectionPool: - # number of pooler instances - connection_pool_number_of_instances: 1 # db schema to install lookup function into connection_pool_schema: "pooler" # db user for pooler to use connection_pool_user: "pooler" # docker image connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer" + # max db connections the pooler should hold + connection_pool_max_db_connections: 60 # default pooling mode connection_pool_mode: "transaction" + # number of pooler instances + connection_pool_number_of_instances: 1 # default resources connection_pool_default_cpu_request: "100m" connection_pool_default_memory_request: "100Mi" diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index a66cc169d..ec79cf751 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -140,6 +140,11 @@ These parameters are grouped directly under the `spec` key in the manifest. is `false`, then no volume will be mounted no matter how operator was configured (so you can override the operator configuration). Optional. +* **enableConnectionPool** + Tells the operator to create a connection pool with a database. If this + field is true, a connection pool deployment will be created even if + `connectionPool` section is empty. Optional, not set by default. + * **enableLogicalBackup** Determines if the logical backup of this cluster should be taken and uploaded to S3. Default: false. Optional. @@ -149,11 +154,6 @@ These parameters are grouped directly under the `spec` key in the manifest. [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule) into account. Optional. Default is: "30 00 \* \* \*" -* enableConnectionPool - Tells the operator to create a connection pool with a database. If this - field is true, a connection pool deployment will be created even if - `connectionPool` section is empty. Optional, not set by default. - ## Postgres parameters Those parameters are grouped under the `postgresql` top-level key, which is @@ -375,9 +375,6 @@ present. * **numberOfInstances** How many instances of connection pool to create. -* **mode** - In which mode to run connection pool, transaction or session. - * **schema** Schema to create for credentials lookup function. @@ -387,5 +384,11 @@ present. * **dockerImage** Which docker image to use for connection pool deployment. +* **maxDBConnections** + How many connections the pooler can max hold. + +* **mode** + In which mode to run connection pool, transaction or session. + * **resources** Resource configuration for connection pool deployment. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 1e8e1a890..a24d5f2f3 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -613,6 +613,9 @@ operator being able to provide some reasonable defaults. * **connection_pool_image** Docker image to use for connection pool deployment. +* **connection_pool_max_db_connections** + How many connections the pooler can max hold. + * **connection_pool_mode** Default pool mode, session or transaction. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index f1c5446b0..a72bb6b73 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -16,8 +16,9 @@ data: # connection_pool_default_memory_limit: 100m # connection_pool_default_memory_request: "100Mi" connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" - # connection_pool_number_of_instances: 1 + # connection_pool_max_db_connections: 60 # connection_pool_mode: "transaction" + # connection_pool_number_of_instances: 1 # connection_pool_schema: "pooler" # connection_pool_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 5bc20eae3..bd1ed6622 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -127,7 +127,8 @@ configuration: connection_pool_default_memory_limit: 100m connection_pool_default_memory_request: "100Mi" connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" - connection_pool_number_of_instances: 1 + # connection_pool_max_db_connections: 60 connection_pool_mode: "transaction" + connection_pool_number_of_instances: 1 # connection_pool_schema: "pooler" # connection_pool_user: "pooler" diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index e775b3336..2e0c9f0a8 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -168,7 +168,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ConnectionPool.Image = util.Coalesce( fromCRD.ConnectionPool.Image, - "pgbouncer:0.0.1") + "registry.opensource.zalan.do/acid/pgbouncer:master-5") result.ConnectionPool.Mode = util.Coalesce( fromCRD.ConnectionPool.Mode, From e645ca5c238879cd830100569d6f8b1df11054f9 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 5 Mar 2020 11:46:19 +0100 Subject: [PATCH 43/61] Prevent original labels from update --- e2e/tests/test_e2e.py | 36 ++++++++++++++++++++++++------------ pkg/cluster/util.go | 14 +++++++------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 92d6ce13e..c553be031 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -75,7 +75,7 @@ def test_enable_load_balancer(self): ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # enable load balancer services pg_patch_enable_lbs = { @@ -123,7 +123,7 @@ def test_min_resource_limits(self): Lower resource limits below configured minimum and let operator fix it ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' _, failover_targets = k8s.get_pg_nodes(cluster_label) # configure minimum boundaries for CPU and memory limits @@ -190,15 +190,26 @@ def test_scaling(self): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - labels = "cluster-name=acid-minimal-cluster" + labels = "application=spilo,cluster-name=acid-minimal-cluster" - k8s.wait_for_pg_to_scale(3) - self.assertEqual(3, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() + try: + k8s.wait_for_pg_to_scale(3) + self.assertEqual(3, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() - k8s.wait_for_pg_to_scale(2) - self.assertEqual(2, k8s.count_pods_with_label(labels)) - self.assert_master_is_unique() + k8s.wait_for_pg_to_scale(2) + self.assertEqual(2, k8s.count_pods_with_label(labels)) + self.assert_master_is_unique() + except timeout_decorator.TimeoutError: + print('Operator log: {}'.format(k8s.get_operator_log())) + pods = k8s.api.core_v1.list_namespaced_pod('default').items + for p in pods: + response = k8s.api.core_v1.read_namespaced_pod( + name=p.metadata.name, + namespace='default' + ) + print('Pod: {}'.format(response)) + raise @timeout_decorator.timeout(TEST_TIMEOUT_SEC) def test_taint_based_eviction(self): @@ -206,7 +217,7 @@ def test_taint_based_eviction(self): Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) current_master_node, failover_targets = k8s.get_pg_nodes(cluster_label) @@ -361,7 +372,8 @@ def test_enable_disable_connection_pool(self): ''' For a database without connection pool, then turns it on, scale up, turn off and on again. Test with different ways of doing this (via - enableConnectionPool or connectionPool configuration section). + enableConnectionPool or connectionPool configuration section). At the + end turn the connection pool off to not interfere with other tests. ''' k8s = self.k8s service_labels = { @@ -560,7 +572,7 @@ def wait_for_pg_to_scale(self, number_of_instances, namespace='default'): _ = self.api.custom_objects_api.patch_namespaced_custom_object( "acid.zalan.do", "v1", namespace, "postgresqls", "acid-minimal-cluster", body) - labels = 'cluster-name=acid-minimal-cluster' + labels = 'application=spilo,cluster-name=acid-minimal-cluster' while self.count_pods_with_label(labels) != number_of_instances: time.sleep(self.RETRY_TIMEOUT_SEC) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index 2f10a1868..fcbc0b0b1 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -420,18 +420,18 @@ func (c *Cluster) labelsSelector() *metav1.LabelSelector { // not interfere with it (it lists all the pods via labels, and if there would // be no difference, it will recreate also pooler pods). func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { - labels := c.labelsSet(false) - connPoolLabels := map[string]string{ + connPoolLabels := labels.Set(map[string]string{}) + + extraLabels := labels.Set(map[string]string{ "connection-pool": c.connPoolName(), "application": "connection-pool", - } + }) - for k, v := range connPoolLabels { - labels[k] = v - } + connPoolLabels = labels.Merge(connPoolLabels, c.labelsSet(false)) + connPoolLabels = labels.Merge(connPoolLabels, extraLabels) return &metav1.LabelSelector{ - MatchLabels: labels, + MatchLabels: connPoolLabels, MatchExpressions: nil, } } From ab118dd78b75128ef5224695cb5fc3814f662372 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 5 Mar 2020 17:31:04 +0100 Subject: [PATCH 44/61] Prevent operator from wrongly syncing pooler user --- pkg/cluster/cluster.go | 2 +- pkg/cluster/sync.go | 44 ++++++++++++++++++++++++++++++++++++---- pkg/cluster/sync_test.go | 2 +- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 6fe94d08a..603011650 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -729,7 +729,7 @@ func (c *Cluster) Update(oldSpec, newSpec *acidv1.Postgresql) error { } // sync connection pool - if err := c.syncConnectionPool(oldSpec, newSpec); err != nil { + if err := c.syncConnectionPool(oldSpec, newSpec, c.installLookupFunction); err != nil { return fmt.Errorf("could not sync connection pool: %v", err) } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 4a38a7e77..27f024c95 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -110,7 +110,7 @@ func (c *Cluster) Sync(newSpec *acidv1.Postgresql) error { } // sync connection pool - if err = c.syncConnectionPool(&oldSpec, newSpec); err != nil { + if err = c.syncConnectionPool(&oldSpec, newSpec, c.installLookupFunction); err != nil { return fmt.Errorf("could not sync connection pool: %v", err) } @@ -464,10 +464,8 @@ func (c *Cluster) syncRoles() (err error) { } if c.needConnectionPool() { - // An exception from system users, connection pool user connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] userNames = append(userNames, connPoolUser.Name) - c.pgUsers[connPoolUser.Name] = connPoolUser } dbUsers, err = c.readPgUsersFromDatabase(userNames) @@ -475,6 +473,20 @@ func (c *Cluster) syncRoles() (err error) { return fmt.Errorf("error getting users from the database: %v", err) } + if c.needConnectionPool() { + connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] + + // An exception from system users, connection pool user should be + // created by operator, but never updated. If connection pool user + // already exist, do not update it. + if _, exist := dbUsers[connPoolUser.Name]; exist { + delete(dbUsers, connPoolUser.Name) + delete(c.pgUsers, connPoolUser.Name) + } else { + c.pgUsers[connPoolUser.Name] = connPoolUser + } + } + pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(dbUsers, c.pgUsers) if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil { return fmt.Errorf("error executing sync statements: %v", err) @@ -603,7 +615,7 @@ func (c *Cluster) syncLogicalBackupJob() error { return nil } -func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error { +func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) @@ -615,6 +627,30 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql) error // in between c.logger.Debug("syncing connection pool") + // in this case also do not forget to install lookup function as for + // creating cluster + if !oldNeedConnPool { + newConnPool := newSpec.Spec.ConnectionPool + + specSchema := "" + specUser := "" + + if newConnPool != nil { + specSchema = newConnPool.Schema + specUser = newConnPool.Schema + } + + schema := util.Coalesce( + specSchema, + c.OpConfig.ConnectionPool.Schema) + + user := util.Coalesce( + specUser, + c.OpConfig.ConnectionPool.User) + + lookup(schema, user) + } + if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { c.logger.Errorf("could not sync connection pool: %v", err) return err diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 33bd01d16..524b24a35 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -201,7 +201,7 @@ func TestConnPoolSynchronization(t *testing.T) { }, } for _, tt := range tests { - err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec) + err := tt.cluster.syncConnectionPool(tt.oldSpec, tt.newSpec, mockInstallLookupFunction) if err := tt.check(tt.cluster, err); err != nil { t.Errorf("%s [%s]: Could not synchronize, %+v", From 80fee17ea4b56d061e3b73338ff6928e1b0a4216 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 6 Mar 2020 13:32:04 +0100 Subject: [PATCH 45/61] Various fixes Sync pool user correctly, without overriding it. Fix numberOfInstances comparison in defaults. Fix maxDBConnections usage. --- pkg/cluster/cluster.go | 21 +++++++++++++++++---- pkg/cluster/k8sres.go | 2 +- pkg/cluster/sync.go | 23 ++++++++--------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 603011650..9a90af23c 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -860,7 +860,6 @@ func (c *Cluster) initSystemUsers() { // Connection pool user is an exception, if requested it's going to be // created by operator as a normal pgUser if c.needConnectionPool() { - // initialize empty connection pool if not done yet if c.Spec.ConnectionPool == nil { c.Spec.ConnectionPool = &acidv1.ConnectionPool{} @@ -870,11 +869,21 @@ func (c *Cluster) initSystemUsers() { c.Spec.ConnectionPool.User, c.OpConfig.ConnectionPool.User) - c.systemUsers[constants.ConnectionPoolUserKeyName] = spec.PgUser{ + // connection pooler application should be able to login with this role + connPoolUser := spec.PgUser{ Origin: spec.RoleConnectionPool, Name: username, + Flags: []string{constants.RoleFlagLogin}, Password: util.RandomPassword(constants.PasswordLength), } + + if _, exists := c.pgUsers[username]; !exists { + c.pgUsers[username] = connPoolUser + } + + if _, exists := c.systemUsers[constants.ConnectionPoolUserKeyName]; !exists { + c.systemUsers[constants.ConnectionPoolUserKeyName] = connPoolUser + } } } @@ -1256,12 +1265,16 @@ func (c *Cluster) needSyncConnPoolDefaults( podTemplate := deployment.Spec.Template poolContainer := podTemplate.Spec.Containers[constants.ConnPoolContainer] + if spec == nil { + spec = &acidv1.ConnectionPool{} + } + if spec.NumberOfInstances == nil && - deployment.Spec.Replicas != config.NumberOfInstances { + *deployment.Spec.Replicas != *config.NumberOfInstances { sync = true msg := fmt.Sprintf("NumberOfInstances is different (%d vs %d)", - deployment.Spec.Replicas, config.NumberOfInstances) + *deployment.Spec.Replicas, *config.NumberOfInstances) reasons = append(reasons, msg) } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 67417f277..333e8aa51 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1803,7 +1803,7 @@ func (c *Cluster) getConnPoolEnvVars(spec *acidv1.PostgresSpec) []v1.EnvVar { }, { Name: "CONNECTION_POOL_MAX_DB_CONN", - Value: fmt.Sprint(effectiveMaxDBConn), + Value: fmt.Sprint(maxDBConn), }, } } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 27f024c95..f3f7d33f3 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -413,12 +413,18 @@ func (c *Cluster) syncSecrets() error { } else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name { secretUsername = constants.ReplicationUserKeyName userMap = c.systemUsers + } else if secretUsername == c.systemUsers[constants.ConnectionPoolUserKeyName].Name { + secretUsername = constants.ConnectionPoolUserKeyName + userMap = c.systemUsers } else { userMap = c.pgUsers } pwdUser := userMap[secretUsername] // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret - if pwdUser.Password != string(secret.Data["password"]) && pwdUser.Origin == spec.RoleOriginInfrastructure { + if pwdUser.Password != string(secret.Data["password"]) && + (pwdUser.Origin == spec.RoleOriginInfrastructure || + pwdUser.Origin == spec.RoleConnectionPool) { + c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name) if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil { return fmt.Errorf("could not update infrastructure role secret for role %q: %v", secretUsername, err) @@ -466,6 +472,7 @@ func (c *Cluster) syncRoles() (err error) { if c.needConnectionPool() { connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] userNames = append(userNames, connPoolUser.Name) + c.pgUsers[connPoolUser.Name] = connPoolUser } dbUsers, err = c.readPgUsersFromDatabase(userNames) @@ -473,20 +480,6 @@ func (c *Cluster) syncRoles() (err error) { return fmt.Errorf("error getting users from the database: %v", err) } - if c.needConnectionPool() { - connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] - - // An exception from system users, connection pool user should be - // created by operator, but never updated. If connection pool user - // already exist, do not update it. - if _, exist := dbUsers[connPoolUser.Name]; exist { - delete(dbUsers, connPoolUser.Name) - delete(c.pgUsers, connPoolUser.Name) - } else { - c.pgUsers[connPoolUser.Name] = connPoolUser - } - } - pgSyncRequests := c.userSyncStrategy.ProduceSyncRequests(dbUsers, c.pgUsers) if err = c.userSyncStrategy.ExecuteSyncRequests(pgSyncRequests, c.pgDb); err != nil { return fmt.Errorf("error executing sync statements: %v", err) From 2e023799e17adcbc8af4b10371d2bccda4ef696b Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 6 Mar 2020 13:44:41 +0100 Subject: [PATCH 46/61] Fix tests To have necessary number of replicas set --- e2e/tests/test_e2e.py | 5 ++--- pkg/cluster/sync_test.go | 1 + pkg/util/k8sutil/k8sutil.go | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py index 0cfa7ca7c..b96dc71cc 100644 --- a/e2e/tests/test_e2e.py +++ b/e2e/tests/test_e2e.py @@ -299,7 +299,7 @@ def test_scaling(self): Scale up from 2 to 3 and back to 2 pods by updating the Postgres manifest at runtime. ''' k8s = self.k8s - labels = "cluster-name=acid-minimal-cluster" + labels = "application=spilo,cluster-name=acid-minimal-cluster" k8s.wait_for_pg_to_scale(3) self.assertEqual(3, k8s.count_pods_with_label(labels)) @@ -441,7 +441,7 @@ def test_taint_based_eviction(self): Add taint "postgres=:NoExecute" to node with master. This must cause a failover. ''' k8s = self.k8s - cluster_label = 'cluster-name=acid-minimal-cluster' + cluster_label = 'application=spilo,cluster-name=acid-minimal-cluster' # get nodes of master and replica(s) (expected target of new master) current_master_node, current_replica_nodes = k8s.get_pg_nodes(cluster_label) @@ -507,7 +507,6 @@ def assert_failover(self, current_master_node, num_replicas, failover_targets, c self.assert_master_is_unique() return new_master_node, new_replica_nodes ->>>>>>> master def assert_master_is_unique(self, namespace='default', clusterName="acid-minimal-cluster"): ''' diff --git a/pkg/cluster/sync_test.go b/pkg/cluster/sync_test.go index 524b24a35..483d4ba58 100644 --- a/pkg/cluster/sync_test.go +++ b/pkg/cluster/sync_test.go @@ -65,6 +65,7 @@ func TestConnPoolSynchronization(t *testing.T) { ConnPoolDefaultCPULimit: "100m", ConnPoolDefaultMemoryRequest: "100Mi", ConnPoolDefaultMemoryLimit: "100Mi", + NumberOfInstances: int32ToPointer(1), }, }, }, k8sutil.KubernetesClient{}, acidv1.Postgresql{}, logger) diff --git a/pkg/util/k8sutil/k8sutil.go b/pkg/util/k8sutil/k8sutil.go index 80d5df0d9..75b99ec7c 100644 --- a/pkg/util/k8sutil/k8sutil.go +++ b/pkg/util/k8sutil/k8sutil.go @@ -288,6 +288,9 @@ func (mock *mockDeployment) Create(*apiappsv1.Deployment) (*apiappsv1.Deployment ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + }, }, nil } @@ -339,6 +342,9 @@ func (mock *mockDeploymentNotExist) Create(*apiappsv1.Deployment) (*apiappsv1.De ObjectMeta: metav1.ObjectMeta{ Name: "test-deployment", }, + Spec: apiappsv1.DeploymentSpec{ + Replicas: Int32ToPointer(1), + }, }, nil } From c0a840c3ed6f8da3440d6fbec7dd0c7cfd6b3dd3 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Mon, 9 Mar 2020 14:05:04 +0100 Subject: [PATCH 47/61] Sync other way around --- pkg/cluster/sync.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index f3f7d33f3..20fb71fd6 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -413,17 +413,13 @@ func (c *Cluster) syncSecrets() error { } else if secretUsername == c.systemUsers[constants.ReplicationUserKeyName].Name { secretUsername = constants.ReplicationUserKeyName userMap = c.systemUsers - } else if secretUsername == c.systemUsers[constants.ConnectionPoolUserKeyName].Name { - secretUsername = constants.ConnectionPoolUserKeyName - userMap = c.systemUsers } else { userMap = c.pgUsers } pwdUser := userMap[secretUsername] // if this secret belongs to the infrastructure role and the password has changed - replace it in the secret if pwdUser.Password != string(secret.Data["password"]) && - (pwdUser.Origin == spec.RoleOriginInfrastructure || - pwdUser.Origin == spec.RoleConnectionPool) { + pwdUser.Origin == spec.RoleOriginInfrastructure { c.logger.Debugf("updating the secret %q from the infrastructure roles", secretSpec.Name) if _, err = c.KubeClient.Secrets(secretSpec.Namespace).Update(secretSpec); err != nil { @@ -472,7 +468,10 @@ func (c *Cluster) syncRoles() (err error) { if c.needConnectionPool() { connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] userNames = append(userNames, connPoolUser.Name) - c.pgUsers[connPoolUser.Name] = connPoolUser + + if _, exists := c.pgUsers[constants.ConnectionPoolUserKeyName]; !exists { + c.pgUsers[connPoolUser.Name] = connPoolUser + } } dbUsers, err = c.readPgUsersFromDatabase(userNames) From 6d1a1ea22cb39d374d4f4cbcf2dde9ede66a4e7f Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 13 Mar 2020 14:34:54 +0100 Subject: [PATCH 48/61] Fix role sync if default pool user/schema changed It requires more accurate lookup function synchronization and couple fixes on the way (e.g. few get rid of using schema from a user secret). For lookup function, since it's idempotend, sync it when we're not sure if it was installed (e.g. when the operator was shutdown and start sync everything at the start) and then remember that it was installed. --- pkg/cluster/cluster.go | 29 +++++++++++++++++++++++++++++ pkg/cluster/database.go | 2 ++ pkg/cluster/k8sres.go | 16 +++++++++++----- pkg/cluster/resources.go | 6 ++---- pkg/cluster/sync.go | 13 ++++++------- 5 files changed, 50 insertions(+), 16 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 9a90af23c..8426cd5f1 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -49,9 +49,18 @@ type Config struct { PodServiceAccountRoleBinding *rbacv1.RoleBinding } +// K8S objects that are belongs to a connection pool type ConnectionPoolObjects struct { Deployment *appsv1.Deployment Service *v1.Service + + // It could happen that a connection pool was enabled, but the operator was + // not able to properly process a corresponding event or was restarted. In + // this case we will miss missing/require situation and a lookup function + // will not be installed. To avoid synchronizing it all the time to prevent + // this, we can remember the result in memory at least until the next + // restart. + LookupFunction bool } type kubeResources struct { @@ -1305,5 +1314,25 @@ func (c *Cluster) needSyncConnPoolDefaults( c.logger.Warningf("Cannot generate expected resources, %v", err) } + for _, env := range poolContainer.Env { + if env.Name == "PGUSER" { + ref := env.ValueFrom.SecretKeyRef.LocalObjectReference + + if ref.Name != c.credentialSecretName(config.User) { + sync = true + msg := fmt.Sprintf("Pool user is different (%s vs %s)", + ref.Name, config.User) + reasons = append(reasons, msg) + } + } + + if env.Name == "PGSCHEMA" && env.Value != config.Schema { + sync = true + msg := fmt.Sprintf("Pool schema is different (%s vs %s)", + env.Value, config.Schema) + reasons = append(reasons, msg) + } + } + return sync, reasons } diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 0c1e07a11..064094680 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -269,6 +269,7 @@ func makeUserFlags(rolsuper, rolinherit, rolcreaterole, rolcreatedb, rolcanlogin // perform remote authentification. func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { var stmtBytes bytes.Buffer + c.logger.Info("Installing lookup function") if err := c.initDbConn(); err != nil { return fmt.Errorf("could not init database connection") @@ -326,5 +327,6 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { c.logger.Infof("Pool lookup function installed into %s", dbname) } + c.ConnectionPool.LookupFunction = true return nil } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index aa4705bb1..c39a3642e 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -1822,14 +1822,22 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( spec.ConnectionPool.DockerImage, c.OpConfig.ConnectionPool.Image) + effectiveSchema := util.Coalesce( + spec.ConnectionPool.Schema, + c.OpConfig.ConnectionPool.Schema) + if err != nil { return nil, fmt.Errorf("could not generate resource requirements: %v", err) } secretSelector := func(key string) *v1.SecretKeySelector { + effectiveUser := util.Coalesce( + spec.ConnectionPool.User, + c.OpConfig.ConnectionPool.User) + return &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ - Name: c.credentialSecretName(c.OpConfig.ConnectionPool.User), + Name: c.credentialSecretName(effectiveUser), }, Key: key, } @@ -1853,10 +1861,8 @@ func (c *Cluster) generateConnPoolPodTemplate(spec *acidv1.PostgresSpec) ( // the convention is to use the same schema name as // connection pool username { - Name: "PGSCHEMA", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: secretSelector("username"), - }, + Name: "PGSCHEMA", + Value: effectiveSchema, }, { Name: "PGPASSWORD", diff --git a/pkg/cluster/resources.go b/pkg/cluster/resources.go index 5c0a7bbd7..2e02a9a83 100644 --- a/pkg/cluster/resources.go +++ b/pkg/cluster/resources.go @@ -188,8 +188,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { return fmt.Errorf("could not delete deployment: %v", err) } - c.logger.Infof("Connection pool deployment %q has been deleted", - util.NameFromMeta(deployment.ObjectMeta)) + c.logger.Infof("Connection pool deployment %q has been deleted", deploymentName) // Repeat the same for the service object service := c.ConnectionPool.Service @@ -211,8 +210,7 @@ func (c *Cluster) deleteConnectionPool() (err error) { return fmt.Errorf("could not delete service: %v", err) } - c.logger.Infof("Connection pool service %q has been deleted", - util.NameFromMeta(deployment.ObjectMeta)) + c.logger.Infof("Connection pool service %q has been deleted", serviceName) c.ConnectionPool = nil return nil diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 20fb71fd6..1b079dd81 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -469,7 +469,7 @@ func (c *Cluster) syncRoles() (err error) { connPoolUser := c.systemUsers[constants.ConnectionPoolUserKeyName] userNames = append(userNames, connPoolUser.Name) - if _, exists := c.pgUsers[constants.ConnectionPoolUserKeyName]; !exists { + if _, exists := c.pgUsers[connPoolUser.Name]; !exists { c.pgUsers[connPoolUser.Name] = connPoolUser } } @@ -608,6 +608,10 @@ func (c *Cluster) syncLogicalBackupJob() error { } func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup InstallFunction) error { + if c.ConnectionPool == nil { + c.ConnectionPool = &ConnectionPoolObjects{} + } + newNeedConnPool := c.needConnectionPoolWorker(&newSpec.Spec) oldNeedConnPool := c.needConnectionPoolWorker(&oldSpec.Spec) @@ -621,7 +625,7 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup // in this case also do not forget to install lookup function as for // creating cluster - if !oldNeedConnPool { + if !oldNeedConnPool || !c.ConnectionPool.LookupFunction { newConnPool := newSpec.Spec.ConnectionPool specSchema := "" @@ -676,11 +680,6 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup // service is missing, create it. After checking, also remember an object for // the future references. func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) error { - if c.ConnectionPool == nil { - c.logger.Warning("Connection pool resources are empty") - c.ConnectionPool = &ConnectionPoolObjects{} - } - deployment, err := c.KubeClient. Deployments(c.Namespace). Get(c.connPoolName(), metav1.GetOptions{}) From cf6541b8cf92fa4457c42d35f0b9c66f3f47159e Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Tue, 17 Mar 2020 11:12:29 +0100 Subject: [PATCH 49/61] Address feedback Small typo-like fixes and proper installing of a lookup function in all the databases. --- pkg/cluster/cluster.go | 12 ++++++------ pkg/cluster/database.go | 24 +++++++++++++++++++++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 8c902afb5..94ca8e864 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1262,7 +1262,7 @@ func syncResources(a, b *v1.ResourceRequirements) bool { return false } -// Check if we need to synchronize connection pool deploymend due to new +// Check if we need to synchronize connection pool deployment due to new // defaults, that are different from what we see in the DeploymentSpec func (c *Cluster) needSyncConnPoolDefaults( spec *acidv1.ConnectionPool, @@ -1283,7 +1283,7 @@ func (c *Cluster) needSyncConnPoolDefaults( *deployment.Spec.Replicas != *config.NumberOfInstances { sync = true - msg := fmt.Sprintf("NumberOfInstances is different (%d vs %d)", + msg := fmt.Sprintf("NumberOfInstances is different (having %d, required %d)", *deployment.Spec.Replicas, *config.NumberOfInstances) reasons = append(reasons, msg) } @@ -1292,7 +1292,7 @@ func (c *Cluster) needSyncConnPoolDefaults( poolContainer.Image != config.Image { sync = true - msg := fmt.Sprintf("DockerImage is different (%s vs %s)", + msg := fmt.Sprintf("DockerImage is different (having %s, required %s)", poolContainer.Image, config.Image) reasons = append(reasons, msg) } @@ -1306,7 +1306,7 @@ func (c *Cluster) needSyncConnPoolDefaults( // updates for new resource values). if err == nil && syncResources(&poolContainer.Resources, expectedResources) { sync = true - msg := fmt.Sprintf("Resources are different (%+v vs %+v)", + msg := fmt.Sprintf("Resources are different (having %+v, required %+v)", poolContainer.Resources, expectedResources) reasons = append(reasons, msg) } @@ -1321,7 +1321,7 @@ func (c *Cluster) needSyncConnPoolDefaults( if ref.Name != c.credentialSecretName(config.User) { sync = true - msg := fmt.Sprintf("Pool user is different (%s vs %s)", + msg := fmt.Sprintf("Pool user is different (having %s, required %s)", ref.Name, config.User) reasons = append(reasons, msg) } @@ -1329,7 +1329,7 @@ func (c *Cluster) needSyncConnPoolDefaults( if env.Name == "PGSCHEMA" && env.Value != config.Schema { sync = true - msg := fmt.Sprintf("Pool schema is different (%s vs %s)", + msg := fmt.Sprintf("Pool schema is different (having %s, required %s)", env.Value, config.Schema) reasons = append(reasons, msg) } diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 064094680..4ad6374a8 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -51,11 +51,16 @@ const ( ` ) -func (c *Cluster) pgConnectionString() string { +func (c *Cluster) pgConnectionString(dbname string) string { password := c.systemUsers[constants.SuperuserKeyName].Password - return fmt.Sprintf("host='%s' dbname=postgres sslmode=require user='%s' password='%s' connect_timeout='%d'", + if dbname == "" { + dbname = "postgres" + } + + return fmt.Sprintf("host='%s' dbname='%s' sslmode=require user='%s' password='%s' connect_timeout='%d'", fmt.Sprintf("%s.%s.svc.%s", c.Name, c.Namespace, c.OpConfig.ClusterDomain), + dbname, c.systemUsers[constants.SuperuserKeyName].Name, strings.Replace(password, "$", "\\$", -1), constants.PostgresConnectTimeout/time.Second) @@ -70,13 +75,17 @@ func (c *Cluster) databaseAccessDisabled() bool { } func (c *Cluster) initDbConn() error { + return c.initDbConnWithName("") +} + +func (c *Cluster) initDbConnWithName(dbname string) error { c.setProcessName("initializing db connection") if c.pgDb != nil { return nil } var conn *sql.DB - connstring := c.pgConnectionString() + connstring := c.pgConnectionString(dbname) finalerr := retryutil.Retry(constants.PostgresConnectTimeout, constants.PostgresConnectRetryTimeout, func() (bool, error) { @@ -275,6 +284,8 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { return fmt.Errorf("could not init database connection") } defer func() { + // in case if everything went fine this can generate a warning about + // trying to close an empty connection. if err := c.closeDbConn(); err != nil { c.logger.Errorf("could not close database connection: %v", err) } @@ -289,6 +300,10 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) for dbname, _ := range currentDatabases { + if err := c.initDbConnWithName(dbname); err != nil { + return fmt.Errorf("could not init database connection") + } + c.logger.Infof("Install pool lookup function into %s", dbname) params := TemplateParams{ @@ -325,6 +340,9 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { } c.logger.Infof("Pool lookup function installed into %s", dbname) + if err := c.closeDbConn(); err != nil { + c.logger.Errorf("could not close database connection: %v", err) + } } c.ConnectionPool.LookupFunction = true From 1c7065e4ce7713cd115e8e374e99661912d05450 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Tue, 17 Mar 2020 11:33:56 +0100 Subject: [PATCH 50/61] Address feedback Rename application for connection pool (ideally in the future make it configurable). Take into accounts nils for MaxInt32 --- pkg/cluster/util.go | 2 +- pkg/util/util.go | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/cluster/util.go b/pkg/cluster/util.go index fcbc0b0b1..dc1e93954 100644 --- a/pkg/cluster/util.go +++ b/pkg/cluster/util.go @@ -424,7 +424,7 @@ func (c *Cluster) connPoolLabelsSelector() *metav1.LabelSelector { extraLabels := labels.Set(map[string]string{ "connection-pool": c.connPoolName(), - "application": "connection-pool", + "application": "db-connection-pool", }) connPoolLabels = labels.Merge(connPoolLabels, c.labelsSet(false)) diff --git a/pkg/util/util.go b/pkg/util/util.go index abb10d42e..c78852726 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -149,7 +149,22 @@ func CoalesceInt32(val, defaultVal *int32) *int32 { return val } +// Return maximum of two integers provided via pointers. If one value is not +// defined, return the other one. If both are not defined, result is also +// undefined, caller needs to check for that. func MaxInt32(a, b *int32) *int32 { + if a == nil && b == nil { + return nil + } + + if a == nil { + return b + } + + if b == nil { + return a + } + if *a > *b { return a } From 48cdbb6a65f1e7355f9ca2ae95f4f4be08a3ef94 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 19 Mar 2020 10:31:58 +0100 Subject: [PATCH 51/61] Minor improvements * Set minimum number of pool instances to 2 * Improve logging of sync reasons * Improve logging of a new pool role --- pkg/cluster/cluster.go | 2 +- pkg/cluster/k8sres.go | 7 +++++++ pkg/cluster/sync.go | 2 +- pkg/spec/types.go | 2 ++ pkg/util/constants/pooler.go | 1 + pkg/util/util.go | 21 ++++++++++++--------- 6 files changed, 24 insertions(+), 11 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 94ca8e864..50cf5392d 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1240,7 +1240,7 @@ func (c *Cluster) needSyncConnPoolSpecs(oldSpec, newSpec *acidv1.ConnectionPool) } for _, change := range changelog { - msg := fmt.Sprintf("%s %+v from %s to %s", + msg := fmt.Sprintf("%s %+v from '%+v' to '%+v'", change.Type, change.Path, change.From, change.To) reasons = append(reasons, msg) } diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 465c39ff6..1235a7e30 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2053,6 +2053,13 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( k8sutil.Int32ToPointer(1)) } + if *numberOfInstances < constants.ConnPoolMinInstances { + msg := "Adjusted number of connection pool instances from %d to %d" + c.logger.Warningf(msg, numberOfInstances, constants.ConnPoolMinInstances) + + *numberOfInstances = constants.ConnPoolMinInstances + } + if err != nil { return nil, err } diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 931e9e5c8..4e101dd9d 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -727,7 +727,7 @@ func (c *Cluster) syncConnectionPoolWorker(oldSpec, newSpec *acidv1.Postgresql) defaultsSync, defaultsReason := c.needSyncConnPoolDefaults(newConnPool, deployment) reason := append(specReason, defaultsReason...) if specSync || defaultsSync { - c.logger.Infof("Update connection pool deployment %s, reason: %s", + c.logger.Infof("Update connection pool deployment %s, reason: %+v", c.connPoolName(), reason) newDeploymentSpec, err := c.generateConnPoolDeployment(&newSpec.Spec) diff --git a/pkg/spec/types.go b/pkg/spec/types.go index 6f071c44a..36783204d 100644 --- a/pkg/spec/types.go +++ b/pkg/spec/types.go @@ -180,6 +180,8 @@ func (r RoleOrigin) String() string { return "teams API role" case RoleOriginSystem: return "system role" + case RoleConnectionPool: + return "connection pool role" default: panic(fmt.Sprintf("bogus role origin value %d", r)) } diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index 0426dbfe8..346299a1c 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -14,4 +14,5 @@ const ( ConnPoolContainer = 0 ConnPoolMaxDBConnections = 60 ConnPoolMaxClientConnections = 10000 + ConnPoolMinInstances = 2 ) diff --git a/pkg/util/util.go b/pkg/util/util.go index c78852726..8e3b7cd21 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -149,22 +149,25 @@ func CoalesceInt32(val, defaultVal *int32) *int32 { return val } +// Test if any of the values is nil +func testNil(values ...*int32) bool { + for _, v := range values { + if v == nil { + return true + } + } + + return false +} + // Return maximum of two integers provided via pointers. If one value is not // defined, return the other one. If both are not defined, result is also // undefined, caller needs to check for that. func MaxInt32(a, b *int32) *int32 { - if a == nil && b == nil { + if testNil(a, b) { return nil } - if a == nil { - return b - } - - if b == nil { - return a - } - if *a > *b { return a } From 20b2fb46420afb5ff10e66b725da4c878b0bd98b Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 19 Mar 2020 11:28:39 +0100 Subject: [PATCH 52/61] Defaults for user/schema fix Verify the defaul values only if the schema doesn't override them. --- pkg/cluster/cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 50cf5392d..dba67c142 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -1316,7 +1316,7 @@ func (c *Cluster) needSyncConnPoolDefaults( } for _, env := range poolContainer.Env { - if env.Name == "PGUSER" { + if spec.User == "" && env.Name == "PGUSER" { ref := env.ValueFrom.SecretKeyRef.LocalObjectReference if ref.Name != c.credentialSecretName(config.User) { @@ -1327,7 +1327,7 @@ func (c *Cluster) needSyncConnPoolDefaults( } } - if env.Name == "PGSCHEMA" && env.Value != config.Schema { + if spec.Schema == "" && env.Name == "PGSCHEMA" && env.Value != config.Schema { sync = true msg := fmt.Sprintf("Pool schema is different (having %s, required %s)", env.Value, config.Schema) From 6ae3c3d752bfc582703c05e713ce0c324a27ab94 Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 19 Mar 2020 12:09:41 +0100 Subject: [PATCH 53/61] Adjust default resource configuration Since connection poolers are usually cpu bounded. --- charts/postgres-operator/values-crd.yaml | 4 ++-- charts/postgres-operator/values.yaml | 4 ++-- pkg/util/config/config.go | 6 +++--- pkg/util/constants/pooler.go | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index df67c7420..903520fad 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -283,9 +283,9 @@ configConnectionPool: # number of pooler instances connection_pool_number_of_instances: 1 # default resources - connection_pool_default_cpu_request: "100m" + connection_pool_default_cpu_request: "1" connection_pool_default_memory_request: "100Mi" - connection_pool_default_cpu_limit: "100m" + connection_pool_default_cpu_limit: "1" connection_pool_default_memory_limit: "100Mi" rbac: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index cd38afbe7..06f1fd322 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -260,9 +260,9 @@ configConnectionPool: # number of pooler instances connection_pool_number_of_instances: 1 # default resources - connection_pool_default_cpu_request: "100m" + connection_pool_default_cpu_request: "1" connection_pool_default_memory_request: "100Mi" - connection_pool_default_cpu_limit: "100m" + connection_pool_default_cpu_limit: "1" connection_pool_default_memory_limit: "100Mi" rbac: diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 5ef7a1875..678dd81fe 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -91,10 +91,10 @@ type ConnectionPool struct { Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer:master-5"` Mode string `name:"connection_pool_mode" default:"transaction"` MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` - ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"100m"` + ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"1"` ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` - ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"3"` - ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"1Gi"` + ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"` + ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"` } // Config describes operator config diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index 346299a1c..affdbc584 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -6,8 +6,8 @@ const ( ConnectionPoolSchemaName = "pooler" ConnectionPoolDefaultType = "pgbouncer" ConnectionPoolDefaultMode = "transaction" - ConnectionPoolDefaultCpuRequest = "100m" - ConnectionPoolDefaultCpuLimit = "100m" + ConnectionPoolDefaultCpuRequest = "1" + ConnectionPoolDefaultCpuLimit = "1" ConnectionPoolDefaultMemoryRequest = "100Mi" ConnectionPoolDefaultMemoryLimit = "100Mi" From f8398066168bc7a03eafe54765d47bbe79e607cc Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Thu, 19 Mar 2020 16:56:04 +0100 Subject: [PATCH 54/61] Address feedback Set default numberOfInstances to 2. Add verifications for config. Fix schema/user typos. Avoid closing an empty connections. --- pkg/cluster/database.go | 26 ++++++++++++++++++++------ pkg/cluster/sync.go | 6 ++++-- pkg/controller/operator_config.go | 4 ++-- pkg/util/config/config.go | 6 ++++++ 4 files changed, 32 insertions(+), 10 deletions(-) diff --git a/pkg/cluster/database.go b/pkg/cluster/database.go index 4ad6374a8..bca68c188 100644 --- a/pkg/cluster/database.go +++ b/pkg/cluster/database.go @@ -44,7 +44,7 @@ const ( $$ LANGUAGE plpgsql SECURITY DEFINER; REVOKE ALL ON FUNCTION {{.pool_schema}}.user_lookup(text) - FROM public, {{.pool_schema}}; + FROM public, {{.pool_user}}; GRANT EXECUTE ON FUNCTION {{.pool_schema}}.user_lookup(text) TO {{.pool_user}}; GRANT USAGE ON SCHEMA {{.pool_schema}} TO {{.pool_user}}; @@ -124,6 +124,10 @@ func (c *Cluster) initDbConnWithName(dbname string) error { return nil } +func (c *Cluster) connectionIsClosed() bool { + return c.pgDb == nil +} + func (c *Cluster) closeDbConn() (err error) { c.setProcessName("closing db connection") if c.pgDb != nil { @@ -284,8 +288,10 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { return fmt.Errorf("could not init database connection") } defer func() { - // in case if everything went fine this can generate a warning about - // trying to close an empty connection. + if c.connectionIsClosed() { + return + } + if err := c.closeDbConn(); err != nil { c.logger.Errorf("could not close database connection: %v", err) } @@ -300,8 +306,12 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { templater := template.Must(template.New("sql").Parse(connectionPoolLookup)) for dbname, _ := range currentDatabases { + if dbname == "template0" || dbname == "template1" { + continue + } + if err := c.initDbConnWithName(dbname); err != nil { - return fmt.Errorf("could not init database connection") + return fmt.Errorf("could not init database connection to %s", dbname) } c.logger.Infof("Install pool lookup function into %s", dbname) @@ -312,8 +322,10 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { } if err := templater.Execute(&stmtBytes, params); err != nil { - return fmt.Errorf("could not prepare sql statement %+v: %v", + c.logger.Errorf("could not prepare sql statement %+v: %v", params, err) + // process other databases + continue } // golang sql will do retries couple of times if pq driver reports @@ -335,8 +347,10 @@ func (c *Cluster) installLookupFunction(poolSchema, poolUser string) error { }) if execErr != nil { - return fmt.Errorf("could not execute after retries %s: %v", + c.logger.Errorf("could not execute after retries %s: %v", stmtBytes.String(), err) + // process other databases + continue } c.logger.Infof("Pool lookup function installed into %s", dbname) diff --git a/pkg/cluster/sync.go b/pkg/cluster/sync.go index 4e101dd9d..a7c933ae7 100644 --- a/pkg/cluster/sync.go +++ b/pkg/cluster/sync.go @@ -645,7 +645,7 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup if newConnPool != nil { specSchema = newConnPool.Schema - specUser = newConnPool.Schema + specUser = newConnPool.User } schema := util.Coalesce( @@ -656,7 +656,9 @@ func (c *Cluster) syncConnectionPool(oldSpec, newSpec *acidv1.Postgresql, lookup specUser, c.OpConfig.ConnectionPool.User) - lookup(schema, user) + if err := lookup(schema, user); err != nil { + return err + } } if err := c.syncConnectionPoolWorker(oldSpec, newSpec); err != nil { diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 663129b6d..3c5bba82f 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -152,11 +152,11 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur // so ensure default values here. result.ConnectionPool.NumberOfInstances = util.CoalesceInt32( fromCRD.ConnectionPool.NumberOfInstances, - int32ToPointer(1)) + int32ToPointer(2)) result.ConnectionPool.NumberOfInstances = util.MaxInt32( result.ConnectionPool.NumberOfInstances, - int32ToPointer(1)) + int32ToPointer(2)) result.ConnectionPool.Schema = util.Coalesce( fromCRD.ConnectionPool.Schema, diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 678dd81fe..31abdb13e 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -8,6 +8,7 @@ import ( "fmt" "github.com/zalando/postgres-operator/pkg/spec" + "github.com/zalando/postgres-operator/pkg/util/constants" ) // CRD describes CustomResourceDefinition specific configuration parameters @@ -211,5 +212,10 @@ func validate(cfg *Config) (err error) { if cfg.Workers == 0 { err = fmt.Errorf("number of workers should be higher than 0") } + + if *cfg.ConnectionPool.NumberOfInstances < constants.ConnPoolMinInstances { + msg := "number of connection pool instances should be higher than %d" + err = fmt.Errorf(msg, constants.ConnPoolMinInstances) + } return } From 7a9d898af7c2135d02389dbbc05e699c6420c8de Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Fri, 20 Mar 2020 09:46:00 +0100 Subject: [PATCH 55/61] Set min number of instances to 2 --- charts/postgres-operator/values-crd.yaml | 2 +- manifests/postgresql-operator-default-configuration.yaml | 2 +- pkg/util/config/config.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 903520fad..0c21ccade 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -281,7 +281,7 @@ configConnectionPool: # default pooling mode connection_pool_mode: "transaction" # number of pooler instances - connection_pool_number_of_instances: 1 + connection_pool_number_of_instances: 2 # default resources connection_pool_default_cpu_request: "1" connection_pool_default_memory_request: "100Mi" diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index be7643892..83f5692de 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -129,6 +129,6 @@ configuration: connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" # connection_pool_max_db_connections: 60 connection_pool_mode: "transaction" - connection_pool_number_of_instances: 1 + connection_pool_number_of_instances: 2 # connection_pool_schema: "pooler" # connection_pool_user: "pooler" diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 31abdb13e..3f8b5b9d5 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -86,7 +86,7 @@ type LogicalBackup struct { // Operator options for connection pooler type ConnectionPool struct { - NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"1"` + NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"` Schema string `name:"connection_pool_schema" default:"pooler"` User string `name:"connection_pool_user" default:"pooler"` Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer:master-5"` From 1ca8028fa5146d16f22652674bbb1e7f006ea98d Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 20 Mar 2020 11:31:49 +0100 Subject: [PATCH 56/61] use min instances 2 everywhere and update reference docs --- .../crds/operatorconfigurations.yaml | 3 ++- charts/postgres-operator/crds/postgresqls.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- docs/reference/operator_parameters.md | 14 +++++++++----- manifests/configmap.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 3 ++- manifests/postgresql.crd.yaml | 2 +- pkg/apis/acid.zalan.do/v1/crds.go | 7 ++++--- 8 files changed, 21 insertions(+), 14 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index a45607905..f34e2957f 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -341,7 +341,8 @@ spec: #default: "transaction" connection_pool_number_of_instances: type: integer - #default: 1 + minimum: 2 + #default: 2 connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' diff --git a/charts/postgres-operator/crds/postgresqls.yaml b/charts/postgres-operator/crds/postgresqls.yaml index d8cfb1556..a4c0e4f3a 100644 --- a/charts/postgres-operator/crds/postgresqls.yaml +++ b/charts/postgres-operator/crds/postgresqls.yaml @@ -120,7 +120,7 @@ spec: - "transaction" numberOfInstances: type: integer - minimum: 1 + minimum: 2 resources: type: object required: diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 06f1fd322..55c9daa6a 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -258,7 +258,7 @@ configConnectionPool: # default pooling mode connection_pool_mode: "transaction" # number of pooler instances - connection_pool_number_of_instances: 1 + connection_pool_number_of_instances: 2 # default resources connection_pool_default_cpu_request: "1" connection_pool_default_memory_request: "100Mi" diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 507f9ad1e..0309d6b68 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -604,25 +604,29 @@ but do not specify some of the parameters. All of them are optional with the operator being able to provide some reasonable defaults. * **connection_pool_number_of_instances** - How many instances of connection pool to create. + How many instances of connection pool to create. Default is 2 which is also + the required minimum. * **connection_pool_schema** - Schema to create for credentials lookup function. + Schema to create for credentials lookup function. Default is `pooler`. * **connection_pool_user** User to create for connection pool to be able to connect to a database. + Default is `pooler`. * **connection_pool_image** Docker image to use for connection pool deployment. + Default: "registry.opensource.zalan.do/acid/pgbouncer:master-5" * **connection_pool_max_db_connections** - How many connections the pooler can max hold. + How many connections the pooler can max hold. Default is 60. * **connection_pool_mode** - Default pool mode, session or transaction. + Default pool mode, `session` or `transaction`. Default is `transaction`. * **connection_pool_default_cpu_request** **connection_pool_default_memory_reques** **connection_pool_default_cpu_limit** **connection_pool_default_memory_limit** - Default resource configuration for connection pool deployment. + Default resource configuration for connection pool deployment. The internal + default for memory limit and request is `100Mi`, for CPU it is `1`. diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6807d39e3..3b248a4dd 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -18,7 +18,7 @@ data: connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" # connection_pool_max_db_connections: 60 # connection_pool_mode: "transaction" - # connection_pool_number_of_instances: 1 + # connection_pool_number_of_instances: 2 # connection_pool_schema: "pooler" # connection_pool_user: "pooler" # custom_service_annotations: "keyx:valuez,keya:valuea" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index aa5db341d..483d5c0cb 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -317,7 +317,8 @@ spec: #default: "transaction" connection_pool_number_of_instances: type: integer - #default: 1 + minimum: 2, + #default: 2 connection_pool_default_cpu_limit: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' diff --git a/manifests/postgresql.crd.yaml b/manifests/postgresql.crd.yaml index ec8fe5d5f..06434da14 100644 --- a/manifests/postgresql.crd.yaml +++ b/manifests/postgresql.crd.yaml @@ -84,7 +84,7 @@ spec: - "transaction" numberOfInstances: type: integer - minimum: 1 + minimum: 2 resources: type: object required: diff --git a/pkg/apis/acid.zalan.do/v1/crds.go b/pkg/apis/acid.zalan.do/v1/crds.go index 2c4491e68..dc552d3f4 100644 --- a/pkg/apis/acid.zalan.do/v1/crds.go +++ b/pkg/apis/acid.zalan.do/v1/crds.go @@ -105,6 +105,7 @@ var OperatorConfigCRDResourceColumns = []apiextv1beta1.CustomResourceColumnDefin var min0 = 0.0 var min1 = 1.0 +var min2 = 2.0 var minDisable = -1.0 // PostgresCRDResourceValidation to check applied manifest parameters @@ -198,7 +199,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ }, "numberOfInstances": { Type: "integer", - Minimum: &min1, + Minimum: &min2, }, "resources": { Type: "object", @@ -491,7 +492,7 @@ var PostgresCRDResourceValidation = apiextv1beta1.CustomResourceValidation{ Type: "string", }, "tls": { - Type: "object", + Type: "object", Required: []string{"secretName"}, Properties: map[string]apiextv1beta1.JSONSchemaProps{ "secretName": { @@ -1166,7 +1167,7 @@ var OperatorConfigCRDResourceValidation = apiextv1beta1.CustomResourceValidation }, "connection_pool_number_of_instances": { Type: "integer", - Minimum: &min1, + Minimum: &min2, }, "connection_pool_schema": { Type: "string", From 0aff65ee6f7b6f74d8084fe2292e4640f48cddc2 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 20 Mar 2020 11:33:33 +0100 Subject: [PATCH 57/61] fix typo --- pkg/cluster/k8sres.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index 1235a7e30..edba7a057 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2071,7 +2071,7 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( Labels: c.labelsSet(true), Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. - // By itself StatefulSet is being deleted with "Ophaned" + // By itself StatefulSet is being deleted with "Orphaned" // propagation policy, which means that it's deletion will not // clean up this deployment, but there is a hope that this object // will be garbage collected if something went wrong and operator @@ -2121,7 +2121,7 @@ func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service Labels: c.labelsSet(true), Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. - // By itself StatefulSet is being deleted with "Ophaned" + // By itself StatefulSet is being deleted with "Orphaned" // propagation policy, which means that it's deletion will not // clean up this service, but there is a hope that this object will // be garbage collected if something went wrong and operator didn't From bdb3eaf9c2cf66dc9b3bd9370ba5816abe784ebd Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 20 Mar 2020 11:40:13 +0100 Subject: [PATCH 58/61] some more minor changes --- charts/postgres-operator/crds/operatorconfigurations.yaml | 2 +- charts/postgres-operator/values-crd.yaml | 4 ++-- charts/postgres-operator/values.yaml | 4 ++-- manifests/configmap.yaml | 4 ++-- manifests/operatorconfiguration.crd.yaml | 4 ++-- manifests/postgresql-operator-default-configuration.yaml | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index f34e2957f..5fadae356 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -354,7 +354,7 @@ spec: connection_pool_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100m" + #default: "100Mi" connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 0c21ccade..5c7e47525 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -284,9 +284,9 @@ configConnectionPool: connection_pool_number_of_instances: 2 # default resources connection_pool_default_cpu_request: "1" - connection_pool_default_memory_request: "100Mi" + connection_pool_default_memory_request: 100Mi connection_pool_default_cpu_limit: "1" - connection_pool_default_memory_limit: "100Mi" + connection_pool_default_memory_limit: 100Mi rbac: # Specifies whether RBAC resources should be created diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 55c9daa6a..5194d02ba 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -261,9 +261,9 @@ configConnectionPool: connection_pool_number_of_instances: 2 # default resources connection_pool_default_cpu_request: "1" - connection_pool_default_memory_request: "100Mi" + connection_pool_default_memory_request: 100Mi connection_pool_default_cpu_limit: "1" - connection_pool_default_memory_limit: "100Mi" + connection_pool_default_memory_limit: 100Mi rbac: # Specifies whether RBAC resources should be created diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 3b248a4dd..6b645dbe6 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -13,8 +13,8 @@ data: cluster_name_label: cluster-name # connection_pool_default_cpu_limit: "1" # connection_pool_default_cpu_request: "1" - # connection_pool_default_memory_limit: 100m - # connection_pool_default_memory_request: "100Mi" + # connection_pool_default_memory_limit: 100Mi + # connection_pool_default_memory_request: 100Mi connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" # connection_pool_max_db_connections: 60 # connection_pool_mode: "transaction" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 483d5c0cb..4b2b977db 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -317,7 +317,7 @@ spec: #default: "transaction" connection_pool_number_of_instances: type: integer - minimum: 2, + minimum: 2 #default: 2 connection_pool_default_cpu_limit: type: string @@ -330,7 +330,7 @@ spec: connection_pool_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' - #default: "100m" + #default: "100Mi" connection_pool_default_memory_request: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 83f5692de..065b50c27 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -124,8 +124,8 @@ configuration: connection_pool: connection_pool_default_cpu_limit: "1" connection_pool_default_cpu_request: "1" - connection_pool_default_memory_limit: 100m - connection_pool_default_memory_request: "100Mi" + connection_pool_default_memory_limit: 100Mi + connection_pool_default_memory_request: 100Mi connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" # connection_pool_max_db_connections: 60 connection_pool_mode: "transaction" From b0f534776d5734c8c15a5e49459f9a497ea8dbf4 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 20 Mar 2020 15:56:27 +0100 Subject: [PATCH 59/61] update pooler default image and add explanation for maxDBConnections param --- charts/postgres-operator/crds/operatorconfigurations.yaml | 2 +- docs/reference/cluster_manifest.md | 3 ++- docs/reference/operator_parameters.md | 6 ++++-- manifests/operatorconfiguration.crd.yaml | 2 +- pkg/controller/operator_config.go | 2 +- pkg/util/config/config.go | 2 +- 6 files changed, 10 insertions(+), 7 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 5fadae356..6c61bfac8 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -329,7 +329,7 @@ spec: #default: "pooler" connection_pool_image: type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + #default: "registry.opensource.zalan.do/acid/pgbouncer" connection_pool_max_db_connections: type: integer #default: 60 diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 34fdd3b70..704a29e99 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -385,7 +385,8 @@ present. Which docker image to use for connection pool deployment. * **maxDBConnections** - How many connections the pooler can max hold. + How many connections the pooler can max hold. This value is divided by the + number of pooler pods. * **mode** In which mode to run connection pool, transaction or session. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 0309d6b68..030ec4f1e 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -616,10 +616,12 @@ operator being able to provide some reasonable defaults. * **connection_pool_image** Docker image to use for connection pool deployment. - Default: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + Default: "registry.opensource.zalan.do/acid/pgbouncer" * **connection_pool_max_db_connections** - How many connections the pooler can max hold. Default is 60. + How many connections the pooler can max hold. This value is divided by the + number of pooler pods. Default is 60 which will make up 30 connections per + pod for the default setup with two instances. * **connection_pool_mode** Default pool mode, `session` or `transaction`. Default is `transaction`. diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 4b2b977db..2b3c2bf09 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -305,7 +305,7 @@ spec: #default: "pooler" connection_pool_image: type: string - #default: "registry.opensource.zalan.do/acid/pgbouncer:master-5" + #default: "registry.opensource.zalan.do/acid/pgbouncer" connection_pool_max_db_connections: type: integer #default: 60 diff --git a/pkg/controller/operator_config.go b/pkg/controller/operator_config.go index 3c5bba82f..970eef701 100644 --- a/pkg/controller/operator_config.go +++ b/pkg/controller/operator_config.go @@ -168,7 +168,7 @@ func (c *Controller) importConfigurationFromCRD(fromCRD *acidv1.OperatorConfigur result.ConnectionPool.Image = util.Coalesce( fromCRD.ConnectionPool.Image, - "registry.opensource.zalan.do/acid/pgbouncer:master-5") + "registry.opensource.zalan.do/acid/pgbouncer") result.ConnectionPool.Mode = util.Coalesce( fromCRD.ConnectionPool.Mode, diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 3f8b5b9d5..73a0d9655 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -89,7 +89,7 @@ type ConnectionPool struct { NumberOfInstances *int32 `name:"connection_pool_number_of_instances" default:"2"` Schema string `name:"connection_pool_schema" default:"pooler"` User string `name:"connection_pool_user" default:"pooler"` - Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer:master-5"` + Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` Mode string `name:"connection_pool_mode" default:"transaction"` MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"1"` From 21af41025af47587be42aad5e8ff1f9b0100bcc5 Mon Sep 17 00:00:00 2001 From: Felix Kunde Date: Fri, 20 Mar 2020 17:26:33 +0100 Subject: [PATCH 60/61] lower CPU request and update docs --- .../postgres-operator/crds/operatorconfigurations.yaml | 2 +- charts/postgres-operator/values-crd.yaml | 2 +- charts/postgres-operator/values.yaml | 2 +- docs/reference/cluster_manifest.md | 4 ++-- docs/reference/operator_parameters.md | 8 ++++---- docs/user.md | 10 +++++----- manifests/configmap.yaml | 2 +- manifests/operatorconfiguration.crd.yaml | 2 +- .../postgresql-operator-default-configuration.yaml | 2 +- pkg/util/config/config.go | 2 +- pkg/util/constants/pooler.go | 2 +- 11 files changed, 19 insertions(+), 19 deletions(-) diff --git a/charts/postgres-operator/crds/operatorconfigurations.yaml b/charts/postgres-operator/crds/operatorconfigurations.yaml index 6c61bfac8..7e3b607c0 100644 --- a/charts/postgres-operator/crds/operatorconfigurations.yaml +++ b/charts/postgres-operator/crds/operatorconfigurations.yaml @@ -350,7 +350,7 @@ spec: connection_pool_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "1" + #default: "500m" connection_pool_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' diff --git a/charts/postgres-operator/values-crd.yaml b/charts/postgres-operator/values-crd.yaml index 5c7e47525..7fd1b5098 100644 --- a/charts/postgres-operator/values-crd.yaml +++ b/charts/postgres-operator/values-crd.yaml @@ -283,7 +283,7 @@ configConnectionPool: # number of pooler instances connection_pool_number_of_instances: 2 # default resources - connection_pool_default_cpu_request: "1" + connection_pool_default_cpu_request: 500m connection_pool_default_memory_request: 100Mi connection_pool_default_cpu_limit: "1" connection_pool_default_memory_limit: 100Mi diff --git a/charts/postgres-operator/values.yaml b/charts/postgres-operator/values.yaml index 5194d02ba..2a1a53fca 100644 --- a/charts/postgres-operator/values.yaml +++ b/charts/postgres-operator/values.yaml @@ -260,7 +260,7 @@ configConnectionPool: # number of pooler instances connection_pool_number_of_instances: 2 # default resources - connection_pool_default_cpu_request: "1" + connection_pool_default_cpu_request: 500m connection_pool_default_memory_request: 100Mi connection_pool_default_cpu_limit: "1" connection_pool_default_memory_limit: 100Mi diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md index 704a29e99..d1fb212ee 100644 --- a/docs/reference/cluster_manifest.md +++ b/docs/reference/cluster_manifest.md @@ -385,8 +385,8 @@ present. Which docker image to use for connection pool deployment. * **maxDBConnections** - How many connections the pooler can max hold. This value is divided by the - number of pooler pods. + How many connections the pooler can max hold. This value is divided among the + pooler pods. * **mode** In which mode to run connection pool, transaction or session. diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md index 030ec4f1e..53cd617c4 100644 --- a/docs/reference/operator_parameters.md +++ b/docs/reference/operator_parameters.md @@ -619,9 +619,9 @@ operator being able to provide some reasonable defaults. Default: "registry.opensource.zalan.do/acid/pgbouncer" * **connection_pool_max_db_connections** - How many connections the pooler can max hold. This value is divided by the - number of pooler pods. Default is 60 which will make up 30 connections per - pod for the default setup with two instances. + How many connections the pooler can max hold. This value is divided among the + pooler pods. Default is 60 which will make up 30 connections per pod for the + default setup with two instances. * **connection_pool_mode** Default pool mode, `session` or `transaction`. Default is `transaction`. @@ -631,4 +631,4 @@ operator being able to provide some reasonable defaults. **connection_pool_default_cpu_limit** **connection_pool_default_memory_limit** Default resource configuration for connection pool deployment. The internal - default for memory limit and request is `100Mi`, for CPU it is `1`. + default for memory request and limit is `100Mi`, for CPU it is `500m` and `1`. diff --git a/docs/user.md b/docs/user.md index b82b55183..acfcb7b54 100644 --- a/docs/user.md +++ b/docs/user.md @@ -535,7 +535,7 @@ To configure a new connection pool, specify: spec: connectionPool: # how many instances of connection pool to create - number_of_instances: 1 + number_of_instances: 2 # in which mode to run, session or transaction mode: "transaction" @@ -550,11 +550,11 @@ spec: # resources for each instance resources: requests: - cpu: "100m" - memory: "100Mi" + cpu: 500m + memory: 100Mi limits: - cpu: "100m" - memory: "100Mi" + cpu: "1" + memory: 100Mi ``` By default `pgbouncer` is used to create a connection pool. To find out about diff --git a/manifests/configmap.yaml b/manifests/configmap.yaml index 6b645dbe6..fdc2d5d56 100644 --- a/manifests/configmap.yaml +++ b/manifests/configmap.yaml @@ -12,7 +12,7 @@ data: cluster_labels: application:spilo cluster_name_label: cluster-name # connection_pool_default_cpu_limit: "1" - # connection_pool_default_cpu_request: "1" + # connection_pool_default_cpu_request: "500m" # connection_pool_default_memory_limit: 100Mi # connection_pool_default_memory_request: 100Mi connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" diff --git a/manifests/operatorconfiguration.crd.yaml b/manifests/operatorconfiguration.crd.yaml index 2b3c2bf09..4e6858af8 100644 --- a/manifests/operatorconfiguration.crd.yaml +++ b/manifests/operatorconfiguration.crd.yaml @@ -326,7 +326,7 @@ spec: connection_pool_default_cpu_request: type: string pattern: '^(\d+m|\d+(\.\d{1,3})?)$' - #default: "1" + #default: "500m" connection_pool_default_memory_limit: type: string pattern: '^(\d+(e\d+)?|\d+(\.\d+)?(e\d+)?[EPTGMK]i?)$' diff --git a/manifests/postgresql-operator-default-configuration.yaml b/manifests/postgresql-operator-default-configuration.yaml index 065b50c27..d4c9b518f 100644 --- a/manifests/postgresql-operator-default-configuration.yaml +++ b/manifests/postgresql-operator-default-configuration.yaml @@ -123,7 +123,7 @@ configuration: # scalyr_server_url: "" connection_pool: connection_pool_default_cpu_limit: "1" - connection_pool_default_cpu_request: "1" + connection_pool_default_cpu_request: "500m" connection_pool_default_memory_limit: 100Mi connection_pool_default_memory_request: 100Mi connection_pool_image: "registry.opensource.zalan.do/acid/pgbouncer:master-5" diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go index 73a0d9655..e0e095617 100644 --- a/pkg/util/config/config.go +++ b/pkg/util/config/config.go @@ -92,7 +92,7 @@ type ConnectionPool struct { Image string `name:"connection_pool_image" default:"registry.opensource.zalan.do/acid/pgbouncer"` Mode string `name:"connection_pool_mode" default:"transaction"` MaxDBConnections *int32 `name:"connection_pool_max_db_connections" default:"60"` - ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"1"` + ConnPoolDefaultCPURequest string `name:"connection_pool_default_cpu_request" default:"500m"` ConnPoolDefaultMemoryRequest string `name:"connection_pool_default_memory_request" default:"100Mi"` ConnPoolDefaultCPULimit string `name:"connection_pool_default_cpu_limit" default:"1"` ConnPoolDefaultMemoryLimit string `name:"connection_pool_default_memory_limit" default:"100Mi"` diff --git a/pkg/util/constants/pooler.go b/pkg/util/constants/pooler.go index affdbc584..540d64e2c 100644 --- a/pkg/util/constants/pooler.go +++ b/pkg/util/constants/pooler.go @@ -6,7 +6,7 @@ const ( ConnectionPoolSchemaName = "pooler" ConnectionPoolDefaultType = "pgbouncer" ConnectionPoolDefaultMode = "transaction" - ConnectionPoolDefaultCpuRequest = "1" + ConnectionPoolDefaultCpuRequest = "500m" ConnectionPoolDefaultCpuLimit = "1" ConnectionPoolDefaultMemoryRequest = "100Mi" ConnectionPoolDefaultMemoryLimit = "100Mi" From 9f51d7377bfcba35b75dceb9e95a24f93da2755e Mon Sep 17 00:00:00 2001 From: Dmitrii Dolgov <9erthalion6@gmail.com> Date: Tue, 24 Mar 2020 16:30:01 +0100 Subject: [PATCH 61/61] Use connection pool labels --- pkg/cluster/k8sres.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/cluster/k8sres.go b/pkg/cluster/k8sres.go index edba7a057..800429930 100644 --- a/pkg/cluster/k8sres.go +++ b/pkg/cluster/k8sres.go @@ -2068,7 +2068,7 @@ func (c *Cluster) generateConnPoolDeployment(spec *acidv1.PostgresSpec) ( ObjectMeta: metav1.ObjectMeta{ Name: c.connPoolName(), Namespace: c.Namespace, - Labels: c.labelsSet(true), + Labels: c.connPoolLabelsSelector().MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned" @@ -2118,7 +2118,7 @@ func (c *Cluster) generateConnPoolService(spec *acidv1.PostgresSpec) *v1.Service ObjectMeta: metav1.ObjectMeta{ Name: c.connPoolName(), Namespace: c.Namespace, - Labels: c.labelsSet(true), + Labels: c.connPoolLabelsSelector().MatchLabels, Annotations: map[string]string{}, // make StatefulSet object its owner to represent the dependency. // By itself StatefulSet is being deleted with "Orphaned"