Skip to content

Commit

Permalink
Fix controller memory usage,
Browse files Browse the repository at this point in the history 
decrease 50x footprint by using filtered cache.

Signed-off-by: zzjin <tczzjin@gmail.com>
  • Loading branch information
@zzjin
zzjin committed Mar 13, 2024
1 parent cbbe279 commit d60e80c
Show file tree
Hide file tree
Showing 16 changed files with 593 additions and 1,808 deletions.
4 changes: 2 additions & 2 deletions controllers/db/adminer/Makefile
View file
Expand Up @@ -118,8 +118,8 @@ CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen
ENVTEST ?= $(LOCALBIN)/setup-envtest

## Tool Versions
KUSTOMIZE_VERSION ?= v5.0.3
CONTROLLER_TOOLS_VERSION ?= v0.12.0
KUSTOMIZE_VERSION ?= v5.3.0
CONTROLLER_TOOLS_VERSION ?= v0.14.0

KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
.PHONY: kustomize
Expand Down
19 changes: 12 additions & 7 deletions controllers/db/adminer/config/crd/bases/adminer.db.sealos.io_adminers.yaml
View file
Expand Up @@ -17,7 +17,7 @@ apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.12.0
controller-gen.kubebuilder.io/version: v0.14.0
name: adminers.adminer.db.sealos.io
spec:
group: adminer.db.sealos.io
Expand Down Expand Up @@ -47,14 +47,19 @@ spec:
description: Adminer is the Schema for the adminers API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
description: |-
APIVersion defines the versioned schema of this representation of an object.
Servers should convert recognized schemas to the latest internal value, and
may reject unrecognized values.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
description: |-
Kind is a string value representing the REST resource this object represents.
Servers may infer this from the endpoint the client submits requests to.
Cannot be updated.
In CamelCase.
More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
type: string
metadata:
type: object
Expand Down
100 changes: 59 additions & 41 deletions controllers/db/adminer/controllers/adminer_controller.go
View file
Expand Up @@ -27,16 +27,19 @@ import (
networkingv1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/cache"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
"sigs.k8s.io/controller-runtime/pkg/log"

adminerv1 "github.com/labring/sealos/controllers/db/adminer/api/v1"
"github.com/labring/sealos/controllers/pkg/utils/label"
)

const (
Expand All @@ -48,6 +51,10 @@ const (
LetterBytes = "abcdefghijklmnopqrstuvwxyz0123456789"
)

const (
AdminerPartOf = "adminer"
)

const (
DefaultDomain = "cloud.sealos.io"
DefaultSecretName = "wildcard-cloud-sealos-io-cert"
Expand Down Expand Up @@ -134,26 +141,32 @@ func (r *AdminerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
return ctrl.Result{}, nil
}

if err := r.syncSecret(ctx, adminer); err != nil {
recLabels := label.RecommendedLabels(&label.Recommended{
Name: adminer.Name,
ManagedBy: label.DefaultManagedBy,
PartOf: AdminerPartOf,
})

if err := r.syncSecret(ctx, adminer, recLabels); err != nil {
logger.Error(err, "create secret failed")
r.recorder.Eventf(adminer, corev1.EventTypeWarning, "Create secret failed", "%v", err)
return ctrl.Result{}, err
}

var hostname string
if err := r.syncDeployment(ctx, adminer, &hostname); err != nil {
if err := r.syncDeployment(ctx, adminer, &hostname, recLabels); err != nil {
logger.Error(err, "create deployment failed")
r.recorder.Eventf(adminer, corev1.EventTypeWarning, "Create deployment failed", "%v", err)
return ctrl.Result{}, err
}

if err := r.syncService(ctx, adminer); err != nil {
if err := r.syncService(ctx, adminer, recLabels); err != nil {
logger.Error(err, "create service failed")
r.recorder.Eventf(adminer, corev1.EventTypeWarning, "Create service failed", "%v", err)
return ctrl.Result{}, err
}

if err := r.syncIngress(ctx, adminer, hostname); err != nil {
if err := r.syncIngress(ctx, adminer, hostname, recLabels); err != nil {
logger.Error(err, "create ingress failed")
r.recorder.Eventf(adminer, corev1.EventTypeWarning, "Create ingress failed", "%v", err)
return ctrl.Result{}, err
Expand Down Expand Up @@ -198,11 +211,12 @@ func (r *AdminerReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct
// })
// }

func (r *AdminerReconciler) syncSecret(ctx context.Context, adminer *adminerv1.Adminer) error {
func (r *AdminerReconciler) syncSecret(ctx context.Context, adminer *adminerv1.Adminer, recLabels map[string]string) error {
secret := &corev1.Secret{
ObjectMeta: metav1.ObjectMeta{
Name: adminer.Name,
Namespace: adminer.Namespace,
Labels: recLabels,
},
}

Expand All @@ -217,20 +231,19 @@ func (r *AdminerReconciler) syncSecret(ctx context.Context, adminer *adminerv1.A
return nil
}

func (r *AdminerReconciler) syncDeployment(ctx context.Context, adminer *adminerv1.Adminer, hostname *string) error {
labelsMap := buildLabelsMap(adminer)

func (r *AdminerReconciler) syncDeployment(ctx context.Context, adminer *adminerv1.Adminer, hostname *string, recLabels map[string]string) error {
objectMeta := metav1.ObjectMeta{
Name: adminer.Name,
Namespace: adminer.Namespace,
Labels: recLabels,
}

selector := &metav1.LabelSelector{
MatchLabels: labelsMap,
MatchLabels: recLabels,
}

templateObjMeta := metav1.ObjectMeta{
Labels: labelsMap,
Labels: recLabels,
}

containers := []corev1.Container{
Expand Down Expand Up @@ -369,40 +382,34 @@ func (r *AdminerReconciler) syncDeployment(ctx context.Context, adminer *adminer
return r.Status().Update(ctx, adminer)
}

func (r *AdminerReconciler) syncService(ctx context.Context, adminer *adminerv1.Adminer) error {
labelsMap := buildLabelsMap(adminer)
expectService := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: adminer.Name,
Namespace: adminer.Namespace,
},
Spec: corev1.ServiceSpec{
Selector: labelsMap,
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{
{Name: "adminer", Port: 8080, TargetPort: intstr.FromInt(8080), Protocol: corev1.ProtocolTCP},
},
func (r *AdminerReconciler) syncService(ctx context.Context, adminer *adminerv1.Adminer, recLabels map[string]string) error {
expectServiceSpec := corev1.ServiceSpec{
Selector: recLabels,
Type: corev1.ServiceTypeClusterIP,
Ports: []corev1.ServicePort{
{Name: "adminer", Port: 8080, TargetPort: intstr.FromInt(8080), Protocol: corev1.ProtocolTCP},
},
}

service := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: adminer.Name,
Namespace: adminer.Namespace,
Labels: recLabels,
},
}

if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, service, func() error {
// only update some specific fields
service.Spec.Selector = expectService.Spec.Selector
service.Spec.Type = expectService.Spec.Type
service.Spec.Selector = expectServiceSpec.Selector
service.Spec.Type = expectServiceSpec.Type
if len(service.Spec.Ports) == 0 {
service.Spec.Ports = expectService.Spec.Ports
service.Spec.Ports = expectServiceSpec.Ports
} else {
service.Spec.Ports[0].Name = expectService.Spec.Ports[0].Name
service.Spec.Ports[0].Port = expectService.Spec.Ports[0].Port
service.Spec.Ports[0].TargetPort = expectService.Spec.Ports[0].TargetPort
service.Spec.Ports[0].Protocol = expectService.Spec.Ports[0].Protocol
service.Spec.Ports[0].Name = expectServiceSpec.Ports[0].Name
service.Spec.Ports[0].Port = expectServiceSpec.Ports[0].Port
service.Spec.Ports[0].TargetPort = expectServiceSpec.Ports[0].TargetPort
service.Spec.Ports[0].Protocol = expectServiceSpec.Ports[0].Protocol
}
return controllerutil.SetControllerReference(adminer, service, r.Scheme)
}); err != nil {
Expand All @@ -411,21 +418,22 @@ func (r *AdminerReconciler) syncService(ctx context.Context, adminer *adminerv1.
return nil
}

func (r *AdminerReconciler) syncIngress(ctx context.Context, adminer *adminerv1.Adminer, hostname string) error {
func (r *AdminerReconciler) syncIngress(ctx context.Context, adminer *adminerv1.Adminer, hostname string, recLabels map[string]string) error {
var err error
host := hostname + "." + r.adminerDomain
switch adminer.Spec.IngressType {
case adminerv1.Nginx:
err = r.syncNginxIngress(ctx, adminer, host)
err = r.syncNginxIngress(ctx, adminer, host, recLabels)
}
return err
}

func (r *AdminerReconciler) syncNginxIngress(ctx context.Context, adminer *adminerv1.Adminer, host string) error {
func (r *AdminerReconciler) syncNginxIngress(ctx context.Context, adminer *adminerv1.Adminer, host string, recLabels map[string]string) error {
ingress := &networkingv1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: adminer.Name,
Namespace: adminer.Namespace,
Labels: recLabels,
},
}
if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, ingress, func() error {
Expand Down Expand Up @@ -481,14 +489,6 @@ func isExpired(adminer *adminerv1.Adminer) bool {
return lastUpdateTime.Add(duration).Before(time.Now())
}

func buildLabelsMap(adminer *adminerv1.Adminer) map[string]string {
labelsMap := map[string]string{
"cloud.sealos.io/app-adminer": adminer.Name,
"app": adminer.Name,
}
return labelsMap
}

func getDomain() string {
domain := os.Getenv("DOMAIN")
if domain == "" {
Expand Down Expand Up @@ -529,6 +529,24 @@ func getSecretNamespace() string {
return secretNamespace
}

func NewCache() cache.NewCacheFunc {
cacheLabelSelector := cache.ObjectSelector{
Label: labels.SelectorFromSet(labels.Set{
label.AppManagedBy: label.DefaultManagedBy,
label.AppPartOf: AdminerPartOf,
}),
}

return cache.BuilderWithOptions(cache.Options{
SelectorsByObject: cache.SelectorsByObject{
&appsv1.Deployment{}: cacheLabelSelector,
&corev1.Service{}: cacheLabelSelector,
&corev1.Secret{}: cacheLabelSelector,
&networkingv1.Ingress{}: cacheLabelSelector,
},
})
}

// SetupWithManager sets up the controller with the Manager.
func (r *AdminerReconciler) SetupWithManager(mgr ctrl.Manager) error {
r.recorder = mgr.GetEventRecorderFor("sealos-db-adminer-controller")
Expand Down
79 changes: 35 additions & 44 deletions controllers/db/adminer/go.mod
View file
Expand Up @@ -4,81 +4,72 @@ go 1.20

require (
github.com/jaevor/go-nanoid v1.3.0
github.com/labring/sealos/controllers/pkg v0.0.0-00010101000000-000000000000
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.27.6
k8s.io/api v0.25.6
k8s.io/apimachinery v0.25.6
k8s.io/client-go v0.25.6
k8s.io/api v0.27.4
k8s.io/apimachinery v0.27.4
k8s.io/client-go v0.27.4
sigs.k8s.io/controller-runtime v0.13.0
)

require (
cloud.google.com/go/compute v1.7.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest v0.11.28 // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.21 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/emicklei/go-restful/v3 v3.10.1 // indirect
github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fsnotify/fsnotify v1.5.4 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-logr/zapr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
github.com/go-openapi/swag v0.22.0 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.1 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.4.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-cmp v0.5.9 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/kr/pretty v0.3.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/prometheus/client_golang v1.13.0 // indirect
github.com/prometheus/client_model v0.2.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/procfs v0.8.0 // indirect
github.com/rogpeppe/go-internal v1.8.0 // indirect
github.com/prometheus/client_golang v1.15.1 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/common v0.44.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa // indirect
golang.org/x/net v0.8.0 // indirect
golang.org/x/oauth2 v0.0.0-20220808172628-8227340efae7 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/net v0.17.0 // indirect
golang.org/x/oauth2 v0.8.0 // indirect
golang.org/x/sys v0.13.0 // indirect
golang.org/x/term v0.13.0 // indirect
golang.org/x/text v0.13.0 // indirect
golang.org/x/time v0.3.0 // indirect
gomodules.xyz/jsonpatch/v2 v2.3.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.28.1 // indirect
google.golang.org/protobuf v1.31.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.25.0 // indirect
k8s.io/component-base v0.25.0 // indirect
k8s.io/klog/v2 v2.70.1 // indirect
k8s.io/kube-openapi v0.0.0-20220803164354-a70c9af30aea // indirect
k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
k8s.io/apiextensions-apiserver v0.27.2 // indirect
k8s.io/component-base v0.27.2 // indirect
k8s.io/klog/v2 v2.100.1 // indirect
k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)

replace github.com/labring/sealos/controllers/pkg => ../../pkg

0 comments on commit d60e80c

Please sign in to comment.