diff --git a/go.mod b/go.mod
index 2379b0a361..96859409f7 100644
--- a/go.mod
+++ b/go.mod
@@ -58,8 +58,10 @@ require (
)
require (
+ github.com/go-test/deep v1.1.0
github.com/minio/kes-go v0.1.0
golang.org/x/mod v0.10.0
+ sigs.k8s.io/controller-runtime v0.13.1
)
require (
@@ -85,6 +87,7 @@ require (
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.10.0 // indirect
github.com/evanphx/json-patch v5.6.0+incompatible // indirect
+ github.com/evanphx/json-patch/v5 v5.6.0 // indirect
github.com/fatih/structs v1.1.0 // indirect
github.com/gdamore/encoding v1.0.0 // indirect
github.com/gdamore/tcell/v2 v2.5.4 // indirect
@@ -188,7 +191,6 @@ require (
k8s.io/apiextensions-apiserver v0.25.4 // indirect
k8s.io/gengo v0.0.0-20220902162205-c0856e24416d // indirect
k8s.io/kube-openapi v0.0.0-20230123231816-1cb3ae25d79a // indirect
- sigs.k8s.io/controller-runtime v0.13.1 // indirect
sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
diff --git a/go.sum b/go.sum
index c5baee2be9..11ed65bf30 100644
--- a/go.sum
+++ b/go.sum
@@ -287,6 +287,8 @@ github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U=
github.com/evanphx/json-patch v5.6.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww=
+github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
@@ -296,6 +298,8 @@ github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo=
github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
+github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko=
github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg=
github.com/gdamore/tcell/v2 v2.5.4 h1:TGU4tSjD3sCL788vFNeJnTdzpNKIw1H5dgLnJRQVv/k=
@@ -316,6 +320,8 @@ github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg
github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0=
github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
+github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
@@ -351,6 +357,8 @@ github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+
github.com/go-openapi/validate v0.22.2-0.20230810035134-348543c76e92 h1:aga9z7JlDIsEmk4rFNxEP1T129jdnzi2eYtG9HIdPR0=
github.com/go-openapi/validate v0.22.2-0.20230810035134-348543c76e92/go.mod h1:kVxh31KbfsxU8ZyoHaDbLBWU5CnMdqBUEtadQ2G4d5M=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg=
+github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE=
github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
@@ -501,6 +509,7 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jedib0t/go-pretty/v6 v6.4.4 h1:N+gz6UngBPF4M288kiMURPHELDMIhF/Em35aYuKrsSc=
github.com/jedib0t/go-pretty/v6 v6.4.4/go.mod h1:MgmISkTWDSFu0xOqiZ0mKNntMQ2mDgOcwOkwBEkMDJI=
+github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI=
github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
@@ -676,11 +685,14 @@ github.com/navidys/tvxwidgets v0.3.0/go.mod h1:Cr8CTnbinH2X8bY/vwb8914mku3qImHQ8
github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo=
github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4=
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/ginkgo/v2 v2.7.0 h1:/XxtEV3I3Eif/HobnVx9YmJgk8ENdRsuUmM+fLCFNow=
github.com/onsi/ginkgo/v2 v2.7.0/go.mod h1:yjiuMwPokqY1XauOgju45q3sJt6VzQ/Fict1LFVcsAo=
github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q=
@@ -1241,6 +1253,8 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T
golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
+gomodules.xyz/jsonpatch/v2 v2.2.0 h1:4pT439QV83L+G9FkcCriY6EkpcK6r6bK+A5FBUMI7qY=
+gomodules.xyz/jsonpatch/v2 v2.2.0/go.mod h1:WXp+iVDkoLQqPudfQ9GBlwB2eZ5DKOnjQZCYdOS8GPY=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
@@ -1470,6 +1484,8 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index c61402a063..41a49cce34 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -37,6 +37,7 @@ import (
promclientset "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
kubeinformers "k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
+ "sigs.k8s.io/controller-runtime/pkg/client"
)
const (
@@ -84,11 +85,15 @@ func StartOperator(kubeconfig string) {
if kubeconfig != "" {
cfg, err = clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
}
-
if err != nil {
klog.Fatalf("Error building kubeconfig: %s", err.Error())
}
+ k8sClient, err := client.New(cfg, client.Options{})
+ if err != nil {
+ klog.Fatalf("Error building k8sClient: %s", err.Error())
+ }
+
kubeClient, err := kubernetes.NewForConfig(cfg)
if err != nil {
klog.Fatalf("Error building Kubernetes clientset: %s", err.Error())
@@ -130,6 +135,7 @@ func StartOperator(kubeconfig string) {
podName,
namespaces,
kubeClient,
+ k8sClient,
controllerClient,
promClient,
kubeInformerFactory.Apps().V1().StatefulSets(),
diff --git a/pkg/controller/main-controller.go b/pkg/controller/main-controller.go
index 426c96d75b..13427a9ed2 100644
--- a/pkg/controller/main-controller.go
+++ b/pkg/controller/main-controller.go
@@ -61,6 +61,7 @@ import (
typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
appslisters "k8s.io/client-go/listers/apps/v1"
corelisters "k8s.io/client-go/listers/core/v1"
+ "sigs.k8s.io/controller-runtime/pkg/client"
promclientset "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned"
@@ -122,6 +123,8 @@ type Controller struct {
podName string
// namespacesToWatch restricts the action of the opreator to a list of namespaces
namespacesToWatch set.StringSet
+ // k8sClient is a kubernetes client
+ k8sClient client.Client
// kubeClientSet is a standard kubernetes clientset
kubeClientSet kubernetes.Interface
// minioClientSet is a clientset for our own API group
@@ -213,7 +216,7 @@ type EventNotification struct {
}
// NewController returns a new sample controller
-func NewController(podName string, namespacesToWatch set.StringSet, kubeClientSet kubernetes.Interface, minioClientSet clientset.Interface, promClient promclientset.Interface, statefulSetInformer appsinformers.StatefulSetInformer, deploymentInformer appsinformers.DeploymentInformer, podInformer coreinformers.PodInformer, tenantInformer informers.TenantInformer, policyBindingInformer stsInformers.PolicyBindingInformer, serviceInformer coreinformers.ServiceInformer, hostsTemplate, operatorVersion string) *Controller {
+func NewController(podName string, namespacesToWatch set.StringSet, kubeClientSet kubernetes.Interface, k8sClient client.Client, minioClientSet clientset.Interface, promClient promclientset.Interface, statefulSetInformer appsinformers.StatefulSetInformer, deploymentInformer appsinformers.DeploymentInformer, podInformer coreinformers.PodInformer, tenantInformer informers.TenantInformer, policyBindingInformer stsInformers.PolicyBindingInformer, serviceInformer coreinformers.ServiceInformer, hostsTemplate, operatorVersion string) *Controller {
// Create event broadcaster
// Add minio-controller types to the default Kubernetes Scheme so Events can be
// logged for minio-controller types.
@@ -246,6 +249,7 @@ func NewController(podName string, namespacesToWatch set.StringSet, kubeClientSe
podName: podName,
namespacesToWatch: namespacesToWatch,
kubeClientSet: kubeClientSet,
+ k8sClient: k8sClient,
minioClientSet: minioClientSet,
promClient: promClient,
statefulSetLister: statefulSetInformer.Lister(),
diff --git a/pkg/controller/pdb.go b/pkg/controller/pdb.go
index 0e8a7e9d5b..4c97a49eea 100644
--- a/pkg/controller/pdb.go
+++ b/pkg/controller/pdb.go
@@ -16,21 +16,19 @@ package controller
import (
"context"
- "encoding/json"
"fmt"
"strings"
"sync"
v2 "github.com/minio/operator/pkg/apis/minio.min.io/v2"
+ "github.com/minio/operator/pkg/runtime"
v1 "k8s.io/api/policy/v1"
"k8s.io/api/policy/v1beta1"
- k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
- "k8s.io/apimachinery/pkg/runtime/schema"
- "k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/klog/v2"
+ "sigs.k8s.io/controller-runtime/pkg/client"
)
// DeletePDB - delete PDB for tenant
@@ -39,33 +37,39 @@ func (c *Controller) DeletePDB(ctx context.Context, t *v2.Tenant) (err error) {
if !available.Available() {
return nil
}
+ listOpt := &client.ListOptions{
+ Namespace: t.Namespace,
+ }
+ client.MatchingLabels{
+ v2.TenantLabel: t.Name,
+ }.ApplyToList(listOpt)
if available.V1Available() {
- err = c.kubeClientSet.PolicyV1().PodDisruptionBudgets(t.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
- LabelSelector: fmt.Sprintf("%s=%s", v2.TenantLabel, t.Name),
- })
+ pdbS := &v1.PodDisruptionBudgetList{}
+ err = c.k8sClient.List(ctx, pdbS, listOpt)
if err != nil {
- // don't exist
- if k8serrors.IsNotFound(err) {
- return nil
- }
- klog.Errorf("Delete tenant %s's V1.PDB failed:%s", t.Name, err.Error())
return err
}
+ for _, item := range pdbS.Items {
+ err = c.k8sClient.Delete(ctx, &item)
+ if err != nil {
+ return err
+ }
+ }
}
if available.V1BetaAvailable() {
- err := c.kubeClientSet.PolicyV1beta1().PodDisruptionBudgets(t.Namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, metav1.ListOptions{
- LabelSelector: fmt.Sprintf("%s=%s", v2.TenantLabel, t.Name),
- })
+ pdbS := &v1beta1.PodDisruptionBudgetList{}
+ err = c.k8sClient.List(ctx, pdbS, listOpt)
if err != nil {
- // don't exist
- if k8serrors.IsNotFound(err) {
- return nil
- }
- klog.Errorf("Delete tenant %s's V1Beta.PDB failed:%s", t.Name, err.Error())
return err
}
+ for _, item := range pdbS.Items {
+ err = c.k8sClient.Delete(ctx, &item)
+ if err != nil {
+ return err
+ }
+ }
}
- return nil
+ return err
}
// CreateOrUpdatePDB - hold PDB as expected
@@ -92,127 +96,48 @@ func (c *Controller) CreateOrUpdatePDB(ctx context.Context, t *v2.Tenant) (err e
return nil
}
}
+ var pdbI client.Object
if available.V1Available() {
- pdbName := t.Name + "-" + pool.Name
- var pdb *v1.PodDisruptionBudget
- var isCreate bool
- pdb, err = c.kubeClientSet.PolicyV1().PodDisruptionBudgets(t.Namespace).Get(ctx, pdbName, metav1.GetOptions{})
- if err != nil {
- if k8serrors.IsNotFound(err) {
- pdb = &v1.PodDisruptionBudget{}
- isCreate = true
- } else {
- return err
- }
- }
- if !isCreate {
- // exist and as expected
- if pdb.Spec.MinAvailable != nil && pdb.Spec.MinAvailable.IntValue() == (int(pool.Servers/2)+1) {
- continue
- }
- }
- // set filed we expected
- pdb.Name = pdbName
- pdb.Namespace = t.Namespace
- minAvailable := intstr.FromInt(int(pool.Servers/2) + 1)
- pdb.Spec.MinAvailable = &minAvailable
- pdb.Labels = map[string]string{
- v2.TenantLabel: t.Name,
- v2.PoolLabel: pool.Name,
- }
- pdb.Spec.Selector = metav1.SetAsLabelSelector(labels.Set{
- v2.TenantLabel: t.Name,
- v2.PoolLabel: pool.Name,
- })
- pdb.OwnerReferences = []metav1.OwnerReference{
- *metav1.NewControllerRef(t, schema.GroupVersionKind{
- Group: v2.SchemeGroupVersion.Group,
- Version: v2.SchemeGroupVersion.Version,
- Kind: v2.MinIOCRDResourceKind,
- }),
- }
- if isCreate {
- _, err = c.kubeClientSet.PolicyV1().PodDisruptionBudgets(t.Namespace).Create(ctx, pdb, metav1.CreateOptions{})
- if err != nil {
- return err
- }
- } else {
- patchData := map[string]interface{}{
- "spec": map[string]interface{}{
- "minAvailable": pdb.Spec.MinAvailable,
- },
- }
- pData, err := json.Marshal(patchData)
- if err != nil {
- return err
- }
- _, err = c.kubeClientSet.PolicyV1().PodDisruptionBudgets(t.Namespace).Patch(ctx, pdbName, types.MergePatchType, pData, metav1.PatchOptions{})
- if err != nil {
- return err
- }
- }
+ pdbI = &v1.PodDisruptionBudget{}
+ } else if available.V1BetaAvailable() {
+ pdbI = &v1beta1.PodDisruptionBudget{}
+ } else {
+ return nil
}
- if available.V1BetaAvailable() {
- pdbName := t.Name + "-" + pool.Name
- var pdb *v1beta1.PodDisruptionBudget
- var isCreate bool
- pdb, err = c.kubeClientSet.PolicyV1beta1().PodDisruptionBudgets(t.Namespace).Get(ctx, pdbName, metav1.GetOptions{})
- if err != nil {
- if k8serrors.IsNotFound(err) {
- pdb = &v1beta1.PodDisruptionBudget{}
- isCreate = true
- } else {
- return err
- }
- }
- if !isCreate {
- // exist and as expected
- if pdb.Spec.MinAvailable != nil && pdb.Spec.MinAvailable.IntValue() == (int(pool.Servers/2)+1) {
- continue
+ pdbI.SetName(t.Name + "-" + pool.Name)
+ pdbI.SetNamespace(t.Namespace)
+ _, err := runtime.NewObjectSyncer(ctx, c.k8sClient, t, func() error {
+ if available.V1Available() {
+ pdb := pdbI.(*v1.PodDisruptionBudget)
+ minAvailable := intstr.FromInt(int(pool.Servers/2) + 1)
+ pdb.Spec.MinAvailable = &minAvailable
+ pdb.Labels = map[string]string{
+ v2.TenantLabel: t.Name,
+ v2.PoolLabel: pool.Name,
}
- }
- // set filed we expected
- pdb.Name = pdbName
- pdb.Namespace = t.Namespace
- minAvailable := intstr.FromInt(int(pool.Servers/2) + 1)
- pdb.Spec.MinAvailable = &minAvailable
- pdb.Labels = map[string]string{
- v2.TenantLabel: t.Name,
- v2.PoolLabel: pool.Name,
- }
- pdb.Spec.Selector = metav1.SetAsLabelSelector(labels.Set{
- v2.TenantLabel: t.Name,
- v2.PoolLabel: pool.Name,
- })
- pdb.OwnerReferences = []metav1.OwnerReference{
- *metav1.NewControllerRef(t, schema.GroupVersionKind{
- Group: v2.SchemeGroupVersion.Group,
- Version: v2.SchemeGroupVersion.Version,
- Kind: v2.MinIOCRDResourceKind,
- }),
- }
- if isCreate {
- _, err = c.kubeClientSet.PolicyV1beta1().PodDisruptionBudgets(t.Namespace).Create(ctx, pdb, metav1.CreateOptions{})
- if err != nil {
- return err
- }
- } else {
- patchData := map[string]interface{}{
- "spec": map[string]interface{}{
- "minAvailable": pdb.Spec.MinAvailable,
- },
- }
- pData, err := json.Marshal(patchData)
- if err != nil {
- return err
- }
- _, err = c.kubeClientSet.PolicyV1beta1().PodDisruptionBudgets(t.Namespace).Patch(ctx, pdbName, types.MergePatchType, pData, metav1.PatchOptions{})
- if err != nil {
- return err
+ pdb.Spec.Selector = metav1.SetAsLabelSelector(labels.Set{
+ v2.TenantLabel: t.Name,
+ v2.PoolLabel: pool.Name,
+ })
+ }
+ if available.V1BetaAvailable() {
+ pdb := pdbI.(*v1beta1.PodDisruptionBudget)
+ minAvailable := intstr.FromInt(int(pool.Servers/2) + 1)
+ pdb.Spec.MinAvailable = &minAvailable
+ pdb.Labels = map[string]string{
+ v2.TenantLabel: t.Name,
+ v2.PoolLabel: pool.Name,
}
+ pdb.Spec.Selector = metav1.SetAsLabelSelector(labels.Set{
+ v2.TenantLabel: t.Name,
+ v2.PoolLabel: pool.Name,
+ })
}
+ return nil
+ }, pdbI, runtime.SyncTypeCreateOrUpdate).Sync(ctx)
+ if err != nil {
+ return err
}
-
}
if len(t.Spec.Pools) == 0 {
return fmt.Errorf("%s empty pools", t.Name)
diff --git a/pkg/runtime/pkg.go b/pkg/runtime/pkg.go
new file mode 100644
index 0000000000..79c192aa25
--- /dev/null
+++ b/pkg/runtime/pkg.go
@@ -0,0 +1,85 @@
+// This file is part of MinIO Operator
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package runtime
+
+import (
+ "context"
+ "fmt"
+ "reflect"
+
+ "k8s.io/apimachinery/pkg/runtime"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+const (
+ eventNormal = "Normal"
+ eventWarning = "Warning"
+)
+
+var (
+ // ErrOwnerDeleted is returned when the object owner is marked for deletion.
+ ErrOwnerDeleted = fmt.Errorf("owner is deleted")
+
+ // ErrIgnore is returned for ignored errors.
+ // Ignored errors are treated by the syncer as successful syncs.
+ ErrIgnore = fmt.Errorf("ignored error")
+)
+
+// SyncType is for controlling syncer performance
+type SyncType string
+
+const (
+ // SyncTypeCreateOrUpdate - if not found will create, if existing will update the object
+ SyncTypeCreateOrUpdate = SyncType("CreateOrUpdate")
+ // SyncTypeCreateOrPatch - if not found will create, if existing will patch the object
+ SyncTypeCreateOrPatch = SyncType("CreateOrPatch")
+ // SyncTypeFoundToUpdate - if not found will do nothing, if existing will update the object
+ SyncTypeFoundToUpdate = SyncType("FoundToUpdate")
+ // SyncTypeFoundToPatch - if not found will do nothing, if existing will patch the object
+ SyncTypeFoundToPatch = SyncType("FoundToUpPatch")
+)
+
+// Syncer is for sync Object's action.
+type Syncer interface {
+ Sync(context.Context) (SyncResult, error)
+ ObjectOwner() runtime.Object
+}
+
+// SyncResult is a result of an Sync.
+type SyncResult struct {
+ Operation controllerutil.OperationResult
+ EventType string
+ EventReason string
+ EventMessage string
+}
+
+// SetEventData sets event data on an SyncResult.
+func (r *SyncResult) SetEventData(eventType, reason, message string) {
+ r.EventType = eventType
+ r.EventReason = reason
+ r.EventMessage = message
+}
+
+// EventReason sets the syncer result reason for kind
+func EventReason(obj client.Object, err error) string {
+ objKindName := reflect.TypeOf(obj).String()
+ if err != nil {
+ return fmt.Sprintf("%sSyncFailed", objKindName)
+ }
+ return fmt.Sprintf("%sSyncSuccessfull", objKindName)
+}
diff --git a/pkg/runtime/sync.go b/pkg/runtime/sync.go
new file mode 100644
index 0000000000..938453ec07
--- /dev/null
+++ b/pkg/runtime/sync.go
@@ -0,0 +1,174 @@
+// This file is part of MinIO Operator
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package runtime
+
+import (
+ "context"
+ "errors"
+ "fmt"
+
+ "github.com/go-test/deep"
+ corev1 "k8s.io/api/core/v1"
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ "k8s.io/apimachinery/pkg/runtime"
+ "k8s.io/klog/v2"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+// ObjectSyncer is a Syncer for sync Objects only by passing a MutateFn.
+type ObjectSyncer struct {
+ Client client.Client
+ Ctx context.Context
+ Obj client.Object
+ Owner client.Object
+ MutateFn controllerutil.MutateFn
+ SyncType SyncType
+ // runtime use that
+ previousObject runtime.Object
+}
+
+var _ Syncer = &ObjectSyncer{}
+
+// NewObjectSyncer creates a new kubernetes.Object syncer for object
+// will set owner. And it can set SyncType. Mostly we should set CreateOrUpdate
+func NewObjectSyncer(ctx context.Context, client client.Client, owner client.Object, MutateFn controllerutil.MutateFn, obj client.Object, syncType SyncType) Syncer {
+ return &ObjectSyncer{
+ Ctx: ctx,
+ Client: client,
+ Obj: obj,
+ Owner: owner,
+ MutateFn: MutateFn,
+ SyncType: syncType,
+ }
+}
+
+// mutateFn Wrap for controllerutil.MutateFn. Do something before create or update or patch.
+func (s *ObjectSyncer) mutateFn() controllerutil.MutateFn {
+ return func() error {
+ s.previousObject = s.Obj.DeepCopyObject()
+ err := s.MutateFn()
+ if err != nil {
+ return err
+ }
+ if s.Owner == nil {
+ return nil
+ }
+ // set owner reference only if owner resource is not being deleted, otherwise the owner
+ // reference will be reset in case of deleting.
+ if s.Owner.GetDeletionTimestamp().IsZero() {
+ if err := controllerutil.SetControllerReference(s.Owner, s.Obj, s.Client.Scheme()); err != nil {
+ return err
+ }
+ } else if ctime := s.Obj.GetCreationTimestamp(); ctime.IsZero() {
+ // the owner is deleted, don't recreate the resource if does not exist, because gc
+ // will not delete it again because has no owner reference set
+ return ErrOwnerDeleted
+ }
+
+ return nil
+ }
+}
+
+// objectTypeName returns the type of Object's Name
+func (s *ObjectSyncer) objectTypeName(obj runtime.Object) string {
+ if obj != nil {
+ gvk, err := apiutil.GVKForObject(obj, s.Client.Scheme())
+ if err != nil {
+ return fmt.Sprintf("%T", obj)
+ }
+ return gvk.String()
+ }
+ return "nil"
+}
+
+// Sync does the actual syncing and implements the Syncer Sync method.
+func (s *ObjectSyncer) Sync(ctx context.Context) (SyncResult, error) {
+ var err error
+ result := SyncResult{}
+ key := client.ObjectKeyFromObject(s.Obj)
+ switch s.SyncType {
+ case SyncTypeCreateOrUpdate:
+ result.Operation, err = controllerutil.CreateOrUpdate(ctx, s.Client, s.Obj, s.mutateFn())
+ case SyncTypeCreateOrPatch:
+ result.Operation, err = controllerutil.CreateOrPatch(ctx, s.Client, s.Obj, s.mutateFn())
+ case SyncTypeFoundToUpdate:
+ // found first
+ key := client.ObjectKeyFromObject(s.Obj)
+ if err := s.Client.Get(ctx, key, s.Obj); err != nil {
+ if apierrors.IsNotFound(err) {
+ result.Operation = controllerutil.OperationResultNone
+ break
+ }
+ }
+ result.Operation, err = controllerutil.CreateOrUpdate(ctx, s.Client, s.Obj, s.mutateFn())
+ case SyncTypeFoundToPatch:
+ // found first
+ key := client.ObjectKeyFromObject(s.Obj)
+ if err := s.Client.Get(ctx, key, s.Obj); err != nil {
+ if apierrors.IsNotFound(err) {
+ result.Operation = controllerutil.OperationResultNone
+ break
+ }
+ }
+ result.Operation, err = controllerutil.CreateOrPatch(ctx, s.Client, s.Obj, s.mutateFn())
+ }
+
+ // get the diff info
+ diff := deep.Equal(redact(s.previousObject), redact(s.Obj))
+ switch {
+ case errors.Is(err, ErrOwnerDeleted):
+ klog.Infof("%s key %s kind %s error %s", string(result.Operation), key, s.objectTypeName(s.Obj), err)
+ err = nil
+ case errors.Is(err, ErrIgnore):
+ klog.Infof("syncer skipped key %s kind %s error %s", key, s.objectTypeName(s.Obj), err)
+ err = nil
+ case err != nil:
+ result.SetEventData(eventWarning, EventReason(s.Obj, err),
+ fmt.Sprintf("%s %s failed syncing: %s", s.objectTypeName(s.Obj), key, err))
+ klog.Errorf("%s key %s kind %s diff %s", string(result.Operation), key, s.objectTypeName(s.Obj), diff)
+ default:
+ result.SetEventData(eventNormal, EventReason(s.Obj, err),
+ fmt.Sprintf("%s %s %s successfully", s.objectTypeName(s.Obj), key, result.Operation))
+ klog.Infof("%s key %s kind %s diff %s", string(result.Operation), key, s.objectTypeName(s.Obj), diff)
+ }
+
+ return result, err
+}
+
+// ObjectOwner returns the ObjectSyncer owner.
+func (s *ObjectSyncer) ObjectOwner() runtime.Object {
+ return s.Owner
+}
+
+// Masking of sensitive information
+func redact(obj runtime.Object) runtime.Object {
+ switch exposed := obj.(type) {
+ case *corev1.Secret:
+ redacted := exposed.DeepCopy()
+ redacted.Data = nil
+ redacted.StringData = nil
+ exposed.ObjectMeta.DeepCopyInto(&redacted.ObjectMeta)
+ return redacted
+ case *corev1.ConfigMap:
+ redacted := exposed.DeepCopy()
+ redacted.Data = nil
+ return redacted
+ }
+ return obj
+}
diff --git a/pkg/runtime/sync_test.go b/pkg/runtime/sync_test.go
new file mode 100644
index 0000000000..04cdef9c5b
--- /dev/null
+++ b/pkg/runtime/sync_test.go
@@ -0,0 +1,202 @@
+// Copyright (C) 2023, MinIO, Inc.
+//
+// This code is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License, version 3,
+// along with this program. If not, see
+
+package runtime
+
+import (
+ "context"
+ "reflect"
+ "testing"
+
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/client-go/kubernetes/scheme"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+ "sigs.k8s.io/controller-runtime/pkg/client/fake"
+ "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+func Test_NewObjectSyncer_CreateOrPatch_Patch(t *testing.T) {
+ // test patch
+ patchReg := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-secret",
+ Namespace: "test-namespace",
+ Annotations: map[string]string{
+ "an1": "vn1",
+ "an2": "vn2",
+ },
+ },
+ Immutable: nil,
+ Data: map[string][]byte{
+ "before": []byte("before"),
+ },
+ }
+ // save the obj
+ cli := fake.NewClientBuilder().WithRuntimeObjects(patchReg).WithScheme(scheme.Scheme).Build()
+ patch := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-secret",
+ Namespace: "test-namespace",
+ },
+ }
+ sync, err := NewObjectSyncer(context.Background(), cli, nil, func() error {
+ if patch.Data == nil {
+ patch.Data = map[string][]byte{}
+ }
+ patch.Data["after"] = []byte("after")
+ if patch.Annotations == nil {
+ patch.Annotations = map[string]string{}
+ }
+ patch.Annotations["an3"] = "vn3"
+ return nil
+ }, patch, SyncTypeCreateOrPatch).Sync(context.Background())
+ if err != nil {
+ return
+ }
+ if sync.Operation != controllerutil.OperationResultUpdated || patch.ResourceVersion == "" {
+ t.Errorf("should make a update call")
+ }
+ if !reflect.DeepEqual(patch.Data, map[string][]byte{
+ "after": []byte("after"),
+ "before": []byte("before"),
+ }) {
+ t.Errorf("patch failed")
+ }
+ if !reflect.DeepEqual(patch.Annotations, map[string]string{
+ "an1": "vn1",
+ "an2": "vn2",
+ "an3": "vn3",
+ }) {
+ t.Errorf("patch failed")
+ }
+ t.Log(sync.EventReason)
+}
+
+func Test_NewObjectSyncer_CreateOrUpdate_Update(t *testing.T) {
+ // test update
+ updateReg := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-secret",
+ Namespace: "test-namespace",
+ Annotations: map[string]string{
+ "an1": "vn1",
+ "an2": "vn2",
+ },
+ },
+ Data: map[string][]byte{
+ "before": []byte("before"),
+ },
+ }
+ // save the obj
+ cli := fake.NewClientBuilder().WithRuntimeObjects(updateReg).WithScheme(scheme.Scheme).Build()
+ update := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-secret",
+ Namespace: "test-namespace",
+ },
+ }
+ sync, err := NewObjectSyncer(context.Background(), cli, nil, func() error {
+ if update.Data == nil {
+ update.Data = map[string][]byte{}
+ }
+ update.Data["after"] = []byte("after")
+ if update.Annotations == nil {
+ update.Annotations = map[string]string{}
+ }
+ update.Annotations["an3"] = "vn3"
+ return nil
+ }, update, SyncTypeCreateOrUpdate).Sync(context.Background())
+ if err != nil {
+ return
+ }
+ if sync.Operation != controllerutil.OperationResultUpdated || update.ResourceVersion == "" {
+ t.Errorf("should make a update call")
+ }
+ if !reflect.DeepEqual(update.Data, map[string][]byte{
+ "after": []byte("after"),
+ "before": []byte("before"),
+ }) {
+ t.Errorf("update failed")
+ }
+ if !reflect.DeepEqual(update.Annotations, map[string]string{
+ "an1": "vn1",
+ "an2": "vn2",
+ "an3": "vn3",
+ }) {
+ t.Errorf("update failed")
+ }
+ t.Log(sync.EventReason)
+}
+
+func Test_NewObjectSyncer_CreateOrUpdate_Create(t *testing.T) {
+ // test create
+ create := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-secret",
+ Namespace: "test-namespace",
+ },
+ }
+ data := map[string][]byte{
+ "create": []byte("create_data"),
+ }
+ cli := fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()
+ sync, err := NewObjectSyncer(context.Background(), cli, nil, func() error {
+ create.Data = data
+ return nil
+ }, create, SyncTypeCreateOrUpdate).Sync(context.Background())
+ if err != nil {
+ return
+ }
+ if sync.Operation != controllerutil.OperationResultCreated || create.ResourceVersion == "" {
+ t.Errorf("should make a create call")
+ }
+ if !reflect.DeepEqual(create.Data, data) {
+ t.Errorf("create failed")
+ }
+ t.Log(sync.EventReason)
+}
+
+type wrapK8sClientCanceledTest struct {
+ client.Client
+}
+
+func (w *wrapK8sClientCanceledTest) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
+ return context.Canceled
+}
+
+func Test_NewObjectSyncer_CreateOrUpdate_CreateCanceled(t *testing.T) {
+ // test create
+ create := &v1.Secret{
+ ObjectMeta: metav1.ObjectMeta{
+ Name: "test-secret",
+ Namespace: "test-namespace",
+ },
+ }
+ data := map[string][]byte{
+ "create": []byte("create_data"),
+ }
+ cli := &wrapK8sClientCanceledTest{fake.NewClientBuilder().WithScheme(scheme.Scheme).Build()}
+ sync, err := NewObjectSyncer(context.Background(), cli, nil, func() error {
+ create.Data = data
+ return nil
+ }, create, SyncTypeCreateOrUpdate).Sync(context.Background())
+ if err == nil {
+ return
+ }
+ if sync.Operation != controllerutil.OperationResultNone || create.ResourceVersion != "" {
+ t.Errorf("shouldn't make a create call")
+ }
+ t.Log(sync.EventReason)
+}