Skip to content

Commit

Permalink
ETCD-519:Add reconcile etcd database size (#1248)
Browse files Browse the repository at this point in the history
* ETCD-519:Add reconcile etcd database size

* bump openshift/api deps

* fix verify deps

* add featureGate accessor

* sorting imports

* fix fmt
  • Loading branch information
Elbehery committed May 7, 2024
1 parent fe27bae commit cbfb856
Show file tree
Hide file tree
Showing 57 changed files with 5,181 additions and 1,471 deletions.
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@ require (
github.com/go-bindata/go-bindata v3.1.2+incompatible
github.com/google/go-cmp v0.6.0
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
github.com/openshift/api v0.0.0-20240412130237-e2b0b690b638
github.com/openshift/api v0.0.0-20240502173326-86b054e342e0
github.com/openshift/build-machinery-go v0.0.0-20231128094528-1e9b1b0595c8
github.com/openshift/client-go v0.0.0-20240405120947-c67c8325cdd8
github.com/openshift/library-go v0.0.0-20240412173449-eb2f24c36528
github.com/openshift/library-go v0.0.0-20240426153406-52527b886e57
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.16.0
github.com/prometheus/common v0.44.0
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -305,14 +305,14 @@ github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4
github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o=
github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg=
github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ=
github.com/openshift/api v0.0.0-20240412130237-e2b0b690b638 h1://6BunjFcTaoaWD9IXRC2BynmIW9ag7k2ekGrUYJbzY=
github.com/openshift/api v0.0.0-20240412130237-e2b0b690b638/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
github.com/openshift/api v0.0.0-20240502173326-86b054e342e0 h1:i50/pgOuycx872nfjip78C45Wswpxy+y5ff/f/iQrEM=
github.com/openshift/api v0.0.0-20240502173326-86b054e342e0/go.mod h1:CxgbWAlvu2iQB0UmKTtRu1YfepRg1/vJ64n2DlIEVz4=
github.com/openshift/build-machinery-go v0.0.0-20231128094528-1e9b1b0595c8 h1:cu3YUMVGsKIyFyJGO3F6BZKGYQZpCKxAv9cBPgQAca8=
github.com/openshift/build-machinery-go v0.0.0-20231128094528-1e9b1b0595c8/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
github.com/openshift/client-go v0.0.0-20240405120947-c67c8325cdd8 h1:HGfbllzRcrJBSiwzNjBCs7sExLUxC5/1evnvlNGB0Cg=
github.com/openshift/client-go v0.0.0-20240405120947-c67c8325cdd8/go.mod h1:+VvvaMSTUhOt+rBq7NwRLSNxq06hTeRCBqm0j0PQEq8=
github.com/openshift/library-go v0.0.0-20240412173449-eb2f24c36528 h1:vnLKZUSW1aPv7Pd6+QYjDUU+/8z2MSBacU38cAlNMPA=
github.com/openshift/library-go v0.0.0-20240412173449-eb2f24c36528/go.mod h1:m/HsttSi90vSixwoy5mPUBHcZid2YRw/QbsLErLxF9s=
github.com/openshift/library-go v0.0.0-20240426153406-52527b886e57 h1:A5yWMIoZBJa4PjODuLUp3TqYUxgg+rMZCRd9TwSHh9Y=
github.com/openshift/library-go v0.0.0-20240426153406-52527b886e57/go.mod h1:lFwyRj0XjUf25Da3Q00y+KuaxCWTJ6YzYPDX1+96nco=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
Expand Down
18 changes: 18 additions & 0 deletions pkg/backendquotahelpers/backend_quota_helpers.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package backendquotahelpers

import (
"fmt"

"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
)

const BackendQuotaFeatureGateName = "EtcdBackendQuota"

func IsBackendQuotaFeatureGateEnabled(featureGateAccessor featuregates.FeatureGateAccess) (bool, error) {
gates, err := featureGateAccessor.CurrentFeatureGates()
if err != nil {
return false, fmt.Errorf("could not access feature gates, error was: %w", err)
}

return gates.Enabled(BackendQuotaFeatureGateName), nil
}
27 changes: 27 additions & 0 deletions pkg/backendquotahelpers/backend_quota_helpers_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
package backendquotahelpers

import (
"testing"

configv1 "github.com/openshift/api/config/v1"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/stretchr/testify/require"
)

func TestBackendQuotaFeatureGateEnabled(t *testing.T) {
backendQuotaFeatureGateAccessor := featuregates.NewHardcodedFeatureGateAccess(
[]configv1.FeatureGateName{BackendQuotaFeatureGateName},
[]configv1.FeatureGateName{})
enabled, err := IsBackendQuotaFeatureGateEnabled(backendQuotaFeatureGateAccessor)
require.True(t, enabled)
require.NoError(t, err)
}

func TestBackendQuotaFeatureGateDisabled(t *testing.T) {
backendQuotaFeatureGateAccessor := featuregates.NewHardcodedFeatureGateAccess(
[]configv1.FeatureGateName{},
[]configv1.FeatureGateName{BackendQuotaFeatureGateName})
enabled, err := IsBackendQuotaFeatureGateEnabled(backendQuotaFeatureGateAccessor)
require.False(t, enabled)
require.NoError(t, err)
}
19 changes: 12 additions & 7 deletions pkg/etcdenvvar/envvarcontroller.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,28 +3,29 @@ package etcdenvvar
import (
"context"
"fmt"
"k8s.io/apimachinery/pkg/labels"
"reflect"
"sync"
"time"

operatorv1 "github.com/openshift/api/operator/v1"
configv1informers "github.com/openshift/client-go/config/informers/externalversions/config/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"

operatorversionedclient "github.com/openshift/client-go/operator/clientset/versioned"
operatorv1listers "github.com/openshift/client-go/operator/listers/operator/v1"
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"

"k8s.io/apimachinery/pkg/labels"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog/v2"

operatorv1 "github.com/openshift/api/operator/v1"

operatorversionedclient "github.com/openshift/client-go/operator/clientset/versioned"
operatorv1listers "github.com/openshift/client-go/operator/listers/operator/v1"
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
)

const workQueueKey = "key"
Expand All @@ -49,6 +50,7 @@ type EnvVarController struct {
masterNodeLister corev1listers.NodeLister
masterNodeLabelSelector labels.Selector
etcdLister operatorv1listers.EtcdLister
featureGateAccessor featuregates.FeatureGateAccess

// queue only ever has one item, but it has nice error handling backoff/retry semantics
queue workqueue.RateLimitingInterface
Expand All @@ -71,6 +73,7 @@ func NewEnvVarController(
networkInformer configv1informers.NetworkInformer,
eventRecorder events.Recorder,
etcdsInformer operatorv1informers.EtcdInformer,
featureGateAccessor featuregates.FeatureGateAccess,
) *EnvVarController {
c := &EnvVarController{
operatorClient: operatorClient,
Expand All @@ -81,6 +84,7 @@ func NewEnvVarController(
masterNodeLabelSelector: masterNodeLabelSelector,
targetImagePullSpec: targetImagePullSpec,
etcdLister: etcdsInformer.Lister(),
featureGateAccessor: featureGateAccessor,

queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "EnvVarController"),
cachesToSync: []cache.InformerSynced{
Expand Down Expand Up @@ -159,6 +163,7 @@ func (c *EnvVarController) checkEnvVars() error {
infrastructureLister: c.infrastructureLister,
networkLister: c.networkLister,
etcdLister: c.etcdLister,
featureGateAccessor: c.featureGateAccessor,
})
if err != nil {
return err
Expand Down
75 changes: 43 additions & 32 deletions pkg/etcdenvvar/envvarcontroller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,52 +8,62 @@ import (
configv1 "github.com/openshift/api/config/v1"
operatorv1 "github.com/openshift/api/operator/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
operatorv1listers "github.com/openshift/client-go/operator/listers/operator/v1"
"github.com/openshift/cluster-etcd-operator/pkg/backendquotahelpers"
"github.com/openshift/cluster-etcd-operator/pkg/operator/ceohelpers"
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
u "github.com/openshift/cluster-etcd-operator/pkg/testutils"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"github.com/openshift/library-go/pkg/operator/events"
"github.com/openshift/library-go/pkg/operator/v1helpers"
"github.com/stretchr/testify/require"
"gopkg.in/yaml.v3"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/client-go/kubernetes/fake"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"

operatorv1listers "github.com/openshift/client-go/operator/listers/operator/v1"
"github.com/stretchr/testify/require"

"gopkg.in/yaml.v3"
)

var defaultEnvResult = map[string]string{
"ALL_ETCD_ENDPOINTS": "https://192.168.2.0:2379,https://192.168.2.1:2379,https://192.168.2.2:2379",
"ETCDCTL_API": "3",
"ETCDCTL_CACERT": "/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt",
"ETCDCTL_CERT": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-NODE_NAME.crt",
"ETCDCTL_ENDPOINTS": "https://192.168.2.0:2379,https://192.168.2.1:2379,https://192.168.2.2:2379",
"ETCDCTL_KEY": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-NODE_NAME.key",
"ETCD_CIPHER_SUITES": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"ETCD_DATA_DIR": "/var/lib/etcd",
"ETCD_ELECTION_TIMEOUT": "1000",
"ETCD_ENABLE_PPROF": "true",
"ETCD_EXPERIMENTAL_MAX_LEARNERS": "1",
"ETCD_EXPERIMENTAL_WARNING_APPLY_DURATION": "200ms",
"ETCD_EXPERIMENTAL_WATCH_PROGRESS_NOTIFY_INTERVAL": "5s",
"ETCD_HEARTBEAT_INTERVAL": "100",
"ETCD_IMAGE": "",
"ETCD_INITIAL_CLUSTER_STATE": "existing",
"ETCD_QUOTA_BACKEND_BYTES": "8589934592",
"ETCD_SOCKET_REUSE_ADDRESS": "true",
"NODE_master_0_ETCD_NAME": "master-0",
"NODE_master_0_ETCD_URL_HOST": "192.168.2.0",
"NODE_master_0_IP": "192.168.2.0",
"NODE_master_1_ETCD_NAME": "master-1",
"NODE_master_1_ETCD_URL_HOST": "192.168.2.1",
"NODE_master_1_IP": "192.168.2.1",
"NODE_master_2_ETCD_NAME": "master-2",
"NODE_master_2_ETCD_URL_HOST": "192.168.2.2",
"NODE_master_2_IP": "192.168.2.2",
}
var (
backendQuotaFeatureGateAccessor = featuregates.NewHardcodedFeatureGateAccess(
[]configv1.FeatureGateName{backendquotahelpers.BackendQuotaFeatureGateName},
[]configv1.FeatureGateName{})

defaultEnvResult = map[string]string{
"ALL_ETCD_ENDPOINTS": "https://192.168.2.0:2379,https://192.168.2.1:2379,https://192.168.2.2:2379",
"ETCDCTL_API": "3",
"ETCDCTL_CACERT": "/etc/kubernetes/static-pod-certs/configmaps/etcd-serving-ca/ca-bundle.crt",
"ETCDCTL_CERT": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-NODE_NAME.crt",
"ETCDCTL_ENDPOINTS": "https://192.168.2.0:2379,https://192.168.2.1:2379,https://192.168.2.2:2379",
"ETCDCTL_KEY": "/etc/kubernetes/static-pod-certs/secrets/etcd-all-certs/etcd-peer-NODE_NAME.key",
"ETCD_CIPHER_SUITES": "TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
"ETCD_DATA_DIR": "/var/lib/etcd",
"ETCD_ELECTION_TIMEOUT": "1000",
"ETCD_ENABLE_PPROF": "true",
"ETCD_EXPERIMENTAL_MAX_LEARNERS": "1",
"ETCD_EXPERIMENTAL_WARNING_APPLY_DURATION": "200ms",
"ETCD_EXPERIMENTAL_WATCH_PROGRESS_NOTIFY_INTERVAL": "5s",
"ETCD_HEARTBEAT_INTERVAL": "100",
"ETCD_IMAGE": "",
"ETCD_INITIAL_CLUSTER_STATE": "existing",
"ETCD_QUOTA_BACKEND_BYTES": "8589934592",
"ETCD_SOCKET_REUSE_ADDRESS": "true",
"NODE_master_0_ETCD_NAME": "master-0",
"NODE_master_0_ETCD_URL_HOST": "192.168.2.0",
"NODE_master_0_IP": "192.168.2.0",
"NODE_master_1_ETCD_NAME": "master-1",
"NODE_master_1_ETCD_URL_HOST": "192.168.2.1",
"NODE_master_1_IP": "192.168.2.1",
"NODE_master_2_ETCD_NAME": "master-2",
"NODE_master_2_ETCD_URL_HOST": "192.168.2.2",
"NODE_master_2_IP": "192.168.2.2",
}
)

func TestEnvVarController(t *testing.T) {
scenarios := []struct {
Expand Down Expand Up @@ -170,6 +180,7 @@ func TestEnvVarController(t *testing.T) {
masterNodeLister: corev1listers.NewNodeLister(indexer),
networkLister: configv1listers.NewNetworkLister(networkIndexer),
etcdLister: operatorv1listers.NewEtcdLister(etcdIndexer),
featureGateAccessor: backendQuotaFeatureGateAccessor,
}

err = controller.sync(context.TODO())
Expand Down
49 changes: 38 additions & 11 deletions pkg/etcdenvvar/etcd_env.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,27 +2,29 @@ package etcdenvvar

import (
"fmt"
"k8s.io/apimachinery/pkg/labels"
"net"
"runtime"
"sort"
"strconv"
"strings"

"github.com/ghodss/yaml"
v1 "github.com/openshift/api/config/v1"
operatorv1 "github.com/openshift/api/operator/v1"
configv1listers "github.com/openshift/client-go/config/listers/config/v1"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/klog/v2"

operatorv1listers "github.com/openshift/client-go/operator/listers/operator/v1"
"github.com/openshift/cluster-etcd-operator/pkg/backendquotahelpers"
"github.com/openshift/cluster-etcd-operator/pkg/dnshelpers"
"github.com/openshift/cluster-etcd-operator/pkg/hwspeedhelpers"
"github.com/openshift/cluster-etcd-operator/pkg/operator/operatorclient"
"github.com/openshift/cluster-etcd-operator/pkg/tlshelpers"
"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"

"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/klog/v2"

"github.com/ghodss/yaml"
)

type envVarContext struct {
Expand All @@ -40,10 +42,9 @@ type envVarContext struct {
}

var FixedEtcdEnvVars = map[string]string{
"ETCD_DATA_DIR": "/var/lib/etcd",
"ETCD_QUOTA_BACKEND_BYTES": "8589934592", // 8 GB
"ETCD_INITIAL_CLUSTER_STATE": "existing",
"ETCD_ENABLE_PPROF": "true",
"ETCD_DATA_DIR": "/var/lib/etcd",
"ETCD_INITIAL_CLUSTER_STATE": "existing",
"ETCD_ENABLE_PPROF": "true",
"ETCD_EXPERIMENTAL_WATCH_PROGRESS_NOTIFY_INTERVAL": "5s",
"ETCD_SOCKET_REUSE_ADDRESS": "true",
"ETCD_EXPERIMENTAL_WARNING_APPLY_DURATION": "200ms",
Expand Down Expand Up @@ -71,6 +72,7 @@ var envVarFns = []envVarFunc{
getAllEtcdEndpoints,
getEtcdctlEnvVars,
getHardwareSpeedValues,
getEtcdDBSize,
getUnsupportedArch,
getCipherSuites,
getMaxLearners,
Expand Down Expand Up @@ -243,6 +245,26 @@ func getHardwareSpeedValues(envVarContext envVarContext) (map[string]string, err
return hwspeedhelpers.HardwareSpeedToEnvMap(speed)
}

func getEtcdDBSize(envVarContext envVarContext) (map[string]string, error) {
etcd, err := envVarContext.etcdLister.Get("cluster")
if err != nil {
return nil, fmt.Errorf("failed to retrieve etcd CR: %v", err)
}

backendQuotaGiB := int64(8)
enabled, err := backendquotahelpers.IsBackendQuotaFeatureGateEnabled(envVarContext.featureGateAccessor)
if err != nil {
return nil, fmt.Errorf("failed check backend quota enabled: %w", err)
}
if enabled && etcd.Spec.BackendQuotaGiB != 0 {
backendQuotaGiB = int64(etcd.Spec.BackendQuotaGiB)
}

return map[string]string{
"ETCD_QUOTA_BACKEND_BYTES": GibibytesToBytesString(backendQuotaGiB),
}, nil
}

func envVarSafe(nodeName string) string {
return strings.ReplaceAll(strings.ReplaceAll(nodeName, "-", "_"), ".", "_")
}
Expand Down Expand Up @@ -336,3 +358,8 @@ func getEtcdEndpoints(configmapLister corev1listers.ConfigMapLister, skipBootstr

return strings.Join(etcdURLs, ","), nil
}

func GibibytesToBytesString(dbSize int64) string {
etcdDBSize := dbSize * 1024 * 1024 * 1024
return strconv.FormatInt(etcdDBSize, 10)
}
47 changes: 47 additions & 0 deletions pkg/etcdenvvar/etcd_env_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
package etcdenvvar

import (
"testing"

"github.com/stretchr/testify/assert"
)

func TestConvertDBSize(t *testing.T) {
testCases := []struct {
name string
input int64
exp string
}{
{
name: "1 GB",
input: 1,
exp: "1073741824",
},
{
name: "8 GB",
input: 8,
exp: "8589934592",
},
{
name: "16 GB",
input: 16,
exp: "17179869184",
},
{
name: "32 GB",
input: 32,
exp: "34359738368",
},
{
name: "64 GB",
input: 64,
exp: "68719476736",
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
assert.Equal(t, tc.exp, GibibytesToBytesString(tc.input))
})
}
}

0 comments on commit cbfb856

Please sign in to comment.