Skip to content

Commit 286c157

Browse files
openshift-merge-robotopenshift-merge-robot
authored
Merge pull request #216 from adambkaplan/ocmo-fix-throttle-2
Bug 1965283: Static Resources Controller for Sync
2 parents c3e81e5 + 0698c47 commit 286c157

86 files changed

Lines changed: 3137 additions & 7973 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎go.mod‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,22 +3,22 @@ module github.com/openshift/cluster-openshift-controller-manager-operator
33
go 1.13
44

55
require (
6+
github.com/fsnotify/fsnotify v1.4.9 // indirect
67
github.com/ghodss/yaml v1.0.0
78
github.com/go-bindata/go-bindata v3.1.2+incompatible
89
github.com/google/gofuzz v1.2.0 // indirect
910
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
1011
github.com/grpc-ecosystem/grpc-gateway v1.11.3 // indirect
1112
github.com/openshift/api v0.0.0-20210416130433-86964261530c
1213
github.com/openshift/build-machinery-go v0.0.0-20210209125900-0da259a2c359
13-
github.com/openshift/client-go v0.0.0-20201020074620-f8fd44879f7c
14-
github.com/openshift/library-go v0.0.0-20201102091359-c4fa0f5b3a08
14+
github.com/openshift/client-go v0.0.0-20210331195552-cf6c2669e01f
15+
github.com/openshift/library-go v0.0.0-20210511143654-b9c317a319e0
1516
github.com/prometheus/client_golang v1.7.1
16-
github.com/spf13/cobra v1.0.0
17+
github.com/spf13/cobra v1.1.1
1718
github.com/spf13/pflag v1.0.5
1819
go.uber.org/zap v1.11.0 // indirect
1920
k8s.io/api v0.21.0
2021
k8s.io/apimachinery v0.21.0
21-
k8s.io/apiserver v0.21.0-rc.0 // indirect
2222
k8s.io/client-go v0.21.0
2323
k8s.io/component-base v0.21.0
2424
k8s.io/klog/v2 v2.8.0

‎go.sum‎

Lines changed: 64 additions & 55 deletions
Large diffs are not rendered by default.

‎pkg/operator/operator.go‎

Lines changed: 13 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -6,18 +6,16 @@ import (
66
"time"
77

88
corev1 "k8s.io/api/core/v1"
9-
"k8s.io/apimachinery/pkg/api/equality"
109
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1110
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1211
"k8s.io/apimachinery/pkg/util/wait"
13-
"k8s.io/client-go/informers"
1412
"k8s.io/client-go/kubernetes"
13+
coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
1514
"k8s.io/client-go/tools/cache"
1615
"k8s.io/client-go/util/flowcontrol"
1716
"k8s.io/client-go/util/workqueue"
1817
"k8s.io/klog/v2"
1918

20-
operatorapiv1 "github.com/openshift/api/operator/v1"
2119
configinformerv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1"
2220
proxyvclient1 "github.com/openshift/client-go/config/listers/config/v1"
2321
operatorclientv1 "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
@@ -38,7 +36,8 @@ type OpenShiftControllerManagerOperator struct {
3836
operatorConfigClient operatorclientv1.OperatorV1Interface
3937
proxyLister proxyvclient1.ProxyLister
4038

41-
kubeClient kubernetes.Interface
39+
kubeClient kubernetes.Interface
40+
configMapsGetter coreclientv1.ConfigMapsGetter
4241

4342
// queue only ever has one item, but it has nice error handling backoff/retry semantics
4443
queue workqueue.RateLimitingInterface
@@ -51,7 +50,7 @@ func NewOpenShiftControllerManagerOperator(
5150
targetImagePullSpec string,
5251
operatorConfigInformer operatorinformersv1.OpenShiftControllerManagerInformer,
5352
proxyInformer configinformerv1.ProxyInformer,
54-
kubeInformersForOpenshiftControllerManager informers.SharedInformerFactory,
53+
kubeInformers v1helpers.KubeInformersForNamespaces,
5554
operatorConfigClient operatorclientv1.OperatorV1Interface,
5655
kubeClient kubernetes.Interface,
5756
recorder events.Recorder,
@@ -61,20 +60,24 @@ func NewOpenShiftControllerManagerOperator(
6160
operatorConfigClient: operatorConfigClient,
6261
proxyLister: proxyInformer.Lister(),
6362
kubeClient: kubeClient,
63+
configMapsGetter: v1helpers.CachedConfigMapGetter(kubeClient.CoreV1(), kubeInformers),
6464
queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "KubeApiserverOperator"),
6565
rateLimiter: flowcontrol.NewTokenBucketRateLimiter(0.05 /*3 per minute*/, 4),
6666
recorder: recorder,
6767
}
6868

6969
operatorConfigInformer.Informer().AddEventHandler(c.eventHandler())
7070
proxyInformer.Informer().AddEventHandler(c.eventHandler())
71-
kubeInformersForOpenshiftControllerManager.Core().V1().ConfigMaps().Informer().AddEventHandler(c.eventHandler())
72-
kubeInformersForOpenshiftControllerManager.Core().V1().ServiceAccounts().Informer().AddEventHandler(c.eventHandler())
73-
kubeInformersForOpenshiftControllerManager.Core().V1().Services().Informer().AddEventHandler(c.eventHandler())
74-
kubeInformersForOpenshiftControllerManager.Apps().V1().Deployments().Informer().AddEventHandler(c.eventHandler())
71+
72+
targetInformers := kubeInformers.InformersFor(util.TargetNamespace)
73+
74+
targetInformers.Core().V1().ConfigMaps().Informer().AddEventHandler(c.eventHandler())
75+
targetInformers.Core().V1().ServiceAccounts().Informer().AddEventHandler(c.eventHandler())
76+
targetInformers.Core().V1().Services().Informer().AddEventHandler(c.eventHandler())
77+
targetInformers.Apps().V1().Deployments().Informer().AddEventHandler(c.eventHandler())
7578

7679
// we only watch some namespaces
77-
kubeInformersForOpenshiftControllerManager.Core().V1().Namespaces().Informer().AddEventHandler(c.namespaceEventHandler())
80+
targetInformers.Core().V1().Namespaces().Informer().AddEventHandler(c.namespaceEventHandler())
7881

7982
// set this bit so the library-go code knows we opt-out from supporting the "unmanaged" state.
8083
management.SetOperatorAlwaysManaged()
@@ -89,35 +92,6 @@ func (c OpenShiftControllerManagerOperator) sync() error {
8992
if err != nil {
9093
return err
9194
}
92-
// manage status
93-
originalOperatorConfig := operatorConfig.DeepCopy()
94-
reasonString := ""
95-
messageString := ""
96-
switch operatorConfig.Spec.ManagementState {
97-
case operatorapiv1.Removed:
98-
fallthrough
99-
// we equally do not allow removed/unmanaged
100-
case operatorapiv1.Unmanaged:
101-
reasonString = fmt.Sprintf("%sUnsupported", string(operatorConfig.Spec.ManagementState))
102-
messageString = fmt.Sprintf("the controller manager spec was set to %s state, but that is unsupported, and has no effect on this condition", string(operatorConfig.Spec.ManagementState))
103-
104-
// as we are ignoring / not supporting unmanaged/removed, we still process any other inputs to the sync/reconciliation
105-
// unlike what we used to do
106-
case operatorapiv1.Managed:
107-
// we want to empty out the reason/message string if transitioning from the other phases so the default setting above is good
108-
}
109-
110-
for _, condition := range operatorConfig.Status.Conditions {
111-
// do not change the current status as part of noting that unmanaged/removed are not supported
112-
condition.Reason = reasonString
113-
condition.Message = messageString
114-
v1helpers.SetOperatorCondition(&operatorConfig.Status.Conditions, condition)
115-
}
116-
if !equality.Semantic.DeepEqual(operatorConfig.Status, originalOperatorConfig.Status) {
117-
if _, err := c.operatorConfigClient.OpenShiftControllerManagers().UpdateStatus(context.TODO(), operatorConfig, metav1.UpdateOptions{}); err != nil {
118-
return err
119-
}
120-
}
12195

12296
forceRequeue, err := syncOpenShiftControllerManager_v311_00_to_latest(c, operatorConfig)
12397
if forceRequeue && err != nil {

‎pkg/operator/starter.go‎

Lines changed: 66 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1010
"k8s.io/client-go/kubernetes"
11+
"k8s.io/client-go/rest"
1112

1213
configv1 "github.com/openshift/api/config/v1"
1314
configclient "github.com/openshift/client-go/config/clientset/versioned"
@@ -16,20 +17,33 @@ import (
1617
operatorclientv1 "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
1718
operatorinformers "github.com/openshift/client-go/operator/informers/externalversions"
1819
"github.com/openshift/library-go/pkg/controller/controllercmd"
20+
"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
1921
"github.com/openshift/library-go/pkg/operator/resourcesynccontroller"
22+
"github.com/openshift/library-go/pkg/operator/staticresourcecontroller"
2023
"github.com/openshift/library-go/pkg/operator/status"
2124
"github.com/openshift/library-go/pkg/operator/v1helpers"
2225

2326
configobservationcontroller "github.com/openshift/cluster-openshift-controller-manager-operator/pkg/operator/configobservation/configobservercontroller"
2427
"github.com/openshift/cluster-openshift-controller-manager-operator/pkg/operator/usercaobservation"
28+
"github.com/openshift/cluster-openshift-controller-manager-operator/pkg/operator/v311_00_assets"
2529
"github.com/openshift/cluster-openshift-controller-manager-operator/pkg/util"
2630
)
2731

2832
func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
29-
kubeClient, err := kubernetes.NewForConfig(controllerConfig.ProtoKubeConfig)
33+
// Increase QPS and burst to avoid client-side rate limits when reconciling RBAC API objects.
34+
// See TODO below for the StaticResourceController
35+
highRateLimitProtoKubeConfig := rest.CopyConfig(controllerConfig.ProtoKubeConfig)
36+
if highRateLimitProtoKubeConfig.QPS < 50 {
37+
highRateLimitProtoKubeConfig.QPS = 50
38+
}
39+
if highRateLimitProtoKubeConfig.Burst < 100 {
40+
highRateLimitProtoKubeConfig.Burst = 100
41+
}
42+
kubeClient, err := kubernetes.NewForConfig(highRateLimitProtoKubeConfig)
3043
if err != nil {
3144
return err
3245
}
46+
3347
operatorClient, err := operatorclient.NewForConfig(controllerConfig.KubeConfig)
3448
if err != nil {
3549
return err
@@ -39,15 +53,26 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
3953
return err
4054
}
4155

42-
kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, util.TargetNamespace, util.OperatorNamespace, util.UserSpecifiedGlobalConfigNamespace)
56+
// Create kube informers for namespaces that the operator reconciles content from or to.
57+
// The empty string "" adds informers for cluster-scoped resources.
58+
kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient,
59+
"",
60+
util.TargetNamespace,
61+
util.OperatorNamespace,
62+
util.UserSpecifiedGlobalConfigNamespace,
63+
util.InfraNamespace,
64+
metav1.NamespaceSystem,
65+
)
4366
operatorConfigInformers := operatorinformers.NewSharedInformerFactory(operatorClient, 10*time.Minute)
4467
configInformers := configinformers.NewSharedInformerFactory(configClient, 10*time.Minute)
4568

69+
// OpenShiftControlllerManagerOperator reconciles the state of the openshift-controller-manager
70+
// DaemonSet and associated ConfigMaps.
4671
operator := NewOpenShiftControllerManagerOperator(
4772
os.Getenv("IMAGE"),
4873
operatorConfigInformers.Operator().V1().OpenShiftControllerManagers(),
4974
configInformers.Config().V1().Proxies(),
50-
kubeInformers.InformersFor(util.TargetNamespace),
75+
kubeInformers,
5176
operatorClient.OperatorV1(),
5277
kubeClient,
5378
controllerConfig.EventRecorder,
@@ -69,6 +94,8 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
6994
controllerConfig.EventRecorder,
7095
)
7196

97+
// ConfigObserver observes the configuration state from cluster config objects and transforms
98+
// them into configuration used by openshift-controller-manager
7299
configObserver := configobservationcontroller.NewConfigObserver(
73100
opClient,
74101
operatorConfigInformers,
@@ -89,6 +116,9 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
89116
openshiftControllerManagers: operatorClient.OperatorV1().OpenShiftControllerManagers(),
90117
version: os.Getenv("RELEASE_VERSION"),
91118
}
119+
120+
// ClusterOperatorStatusController aggregates the conditions in our openshiftcontrollermanager
121+
// object to the corresponding ClusterOperator object.
92122
clusterOperatorStatus := status.NewClusterOperatorStatusController(
93123
util.ClusterOperatorName,
94124
[]configv1.ObjectReference{
@@ -105,10 +135,43 @@ func RunOperator(ctx context.Context, controllerConfig *controllercmd.Controller
105135
controllerConfig.EventRecorder,
106136
)
107137

138+
// StaticResourceController uses library-go's resourceapply package to reconcile a set of YAML
139+
// manifests against a cluster.
140+
// TODO: enhance resourceapply to use listers for RBAC APIs.
141+
staticResourceController := staticresourcecontroller.NewStaticResourceController(
142+
"OpenshiftControllerManagerStaticResources",
143+
v311_00_assets.Asset,
144+
[]string{
145+
"v3.11.0/openshift-controller-manager/informer-clusterrole.yaml",
146+
"v3.11.0/openshift-controller-manager/informer-clusterrolebinding.yaml",
147+
"v3.11.0/openshift-controller-manager/ingress-to-route-controller-clusterrole.yaml",
148+
"v3.11.0/openshift-controller-manager/ingress-to-route-controller-clusterrolebinding.yaml",
149+
"v3.11.0/openshift-controller-manager/tokenreview-clusterrole.yaml",
150+
"v3.11.0/openshift-controller-manager/tokenreview-clusterrolebinding.yaml",
151+
"v3.11.0/openshift-controller-manager/leader-role.yaml",
152+
"v3.11.0/openshift-controller-manager/leader-rolebinding.yaml",
153+
"v3.11.0/openshift-controller-manager/ns.yaml",
154+
"v3.11.0/openshift-controller-manager/old-leader-role.yaml",
155+
"v3.11.0/openshift-controller-manager/old-leader-rolebinding.yaml",
156+
"v3.11.0/openshift-controller-manager/separate-sa-role.yaml",
157+
"v3.11.0/openshift-controller-manager/separate-sa-rolebinding.yaml",
158+
"v3.11.0/openshift-controller-manager/sa.yaml",
159+
"v3.11.0/openshift-controller-manager/svc.yaml",
160+
"v3.11.0/openshift-controller-manager/servicemonitor-role.yaml",
161+
"v3.11.0/openshift-controller-manager/servicemonitor-rolebinding.yaml",
162+
"v3.11.0/openshift-controller-manager/buildconfigstatus-clusterrole.yaml",
163+
"v3.11.0/openshift-controller-manager/buildconfigstatus-clusterrolebinding.yaml",
164+
},
165+
resourceapply.NewKubeClientHolder(kubeClient),
166+
opClient,
167+
controllerConfig.EventRecorder,
168+
).AddKubeInformers(kubeInformers)
169+
108170
operatorConfigInformers.Start(ctx.Done())
109171
kubeInformers.Start(ctx.Done())
110172
configInformers.Start(ctx.Done())
111173

174+
go staticResourceController.Run(ctx, 1)
112175
go operator.Run(ctx, 1)
113176
go resourceSyncer.Run(ctx, 1)
114177
go configObserver.Run(ctx, 1)

‎pkg/operator/sync_openshiftcontrollermanager_v311_00.go‎

Lines changed: 5 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,6 @@ import (
1212
"k8s.io/apimachinery/pkg/api/equality"
1313
apierrors "k8s.io/apimachinery/pkg/api/errors"
1414
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
15-
"k8s.io/apimachinery/pkg/util/sets"
1615
"k8s.io/client-go/kubernetes"
1716
appsclientv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
1817
coreclientv1 "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -38,43 +37,11 @@ func syncOpenShiftControllerManager_v311_00_to_latest(c OpenShiftControllerManag
3837
errors := []error{}
3938
var err error
4039
operatorConfig := originalOperatorConfig.DeepCopy()
41-
clientHolder := resourceapply.NewKubeClientHolder(c.kubeClient)
42-
directResourceResults := resourceapply.ApplyDirectly(clientHolder, c.recorder, v311_00_assets.Asset,
43-
"v3.11.0/openshift-controller-manager/informer-clusterrole.yaml",
44-
"v3.11.0/openshift-controller-manager/informer-clusterrolebinding.yaml",
45-
"v3.11.0/openshift-controller-manager/ingress-to-route-controller-clusterrole.yaml",
46-
"v3.11.0/openshift-controller-manager/ingress-to-route-controller-clusterrolebinding.yaml",
47-
"v3.11.0/openshift-controller-manager/tokenreview-clusterrole.yaml",
48-
"v3.11.0/openshift-controller-manager/tokenreview-clusterrolebinding.yaml",
49-
"v3.11.0/openshift-controller-manager/leader-role.yaml",
50-
"v3.11.0/openshift-controller-manager/leader-rolebinding.yaml",
51-
"v3.11.0/openshift-controller-manager/ns.yaml",
52-
"v3.11.0/openshift-controller-manager/old-leader-role.yaml",
53-
"v3.11.0/openshift-controller-manager/old-leader-rolebinding.yaml",
54-
"v3.11.0/openshift-controller-manager/separate-sa-role.yaml",
55-
"v3.11.0/openshift-controller-manager/separate-sa-rolebinding.yaml",
56-
"v3.11.0/openshift-controller-manager/sa.yaml",
57-
"v3.11.0/openshift-controller-manager/svc.yaml",
58-
"v3.11.0/openshift-controller-manager/servicemonitor-role.yaml",
59-
"v3.11.0/openshift-controller-manager/servicemonitor-rolebinding.yaml",
60-
"v3.11.0/openshift-controller-manager/buildconfigstatus-clusterrole.yaml",
61-
"v3.11.0/openshift-controller-manager/buildconfigstatus-clusterrolebinding.yaml",
62-
)
63-
resourcesThatForceRedeployment := sets.NewString("v3.11.0/openshift-controller-manager/sa.yaml")
64-
forceRollout := false
65-
66-
for _, currResult := range directResourceResults {
67-
if currResult.Error != nil {
68-
errors = append(errors, fmt.Errorf("%q (%T): %v", currResult.File, currResult.Type, currResult.Error))
69-
continue
70-
}
7140

72-
if currResult.Changed && resourcesThatForceRedeployment.Has(currResult.File) {
73-
forceRollout = true
74-
}
75-
}
41+
// TODO - use labels/annotations to force a daemonset rollout
42+
forceRollout := false
7643

77-
_, configMapModified, err := manageOpenShiftControllerManagerConfigMap_v311_00_to_latest(c.kubeClient, c.kubeClient.CoreV1(), c.recorder, operatorConfig)
44+
_, configMapModified, err := manageOpenShiftControllerManagerConfigMap_v311_00_to_latest(c.kubeClient, c.configMapsGetter, c.recorder, operatorConfig)
7845
if err != nil {
7946
errors = append(errors, fmt.Errorf("%q: %v", "configmap", err))
8047
}
@@ -84,12 +51,12 @@ func syncOpenShiftControllerManager_v311_00_to_latest(c OpenShiftControllerManag
8451
errors = append(errors, fmt.Errorf("%q: %v", "client-ca", err))
8552
}
8653

87-
_, serviceCAModified, err := manageOpenShiftServiceCAConfigMap_v311_00_to_latest(c.kubeClient, c.kubeClient.CoreV1(), c.recorder)
54+
_, serviceCAModified, err := manageOpenShiftServiceCAConfigMap_v311_00_to_latest(c.kubeClient, c.configMapsGetter, c.recorder)
8855
if err != nil {
8956
errors = append(errors, fmt.Errorf("%q: %v", "openshift-service-ca", err))
9057
}
9158

92-
_, globalCAModified, err := manageOpenShiftGlobalCAConfigMap_v311_00_to_latest(c.kubeClient, c.kubeClient.CoreV1(), c.recorder)
59+
_, globalCAModified, err := manageOpenShiftGlobalCAConfigMap_v311_00_to_latest(c.kubeClient, c.configMapsGetter, c.recorder)
9360
if err != nil {
9461
errors = append(errors, fmt.Errorf("%q: %v", "openshift-global-ca", err))
9562
}

‎pkg/operator/sync_openshiftcontrollermanager_v311_00_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -214,6 +214,7 @@ func TestProgressingCondition(t *testing.T) {
214214

215215
operator := OpenShiftControllerManagerOperator{
216216
kubeClient: kubeClient,
217+
configMapsGetter: kubeClient.CoreV1(),
217218
proxyLister: proxyLister,
218219
recorder: events.NewInMemoryRecorder(""),
219220
operatorConfigClient: controllerManagerOperatorClient.OperatorV1(),

‎pkg/util/consts.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ const (
66
MachineSpecifiedGlobalConfigNamespace = "openshift-config-managed"
77
TargetNamespace = "openshift-controller-manager"
88
OperatorNamespace = "openshift-controller-manager-operator"
9+
InfraNamespace = "openshift-infra"
910
VersionAnnotation = "release.openshift.io/version"
1011
ClusterOperatorName = "openshift-controller-manager"
1112
)

‎vendor/github.com/certifi/gocertifi/LICENSE‎

Lines changed: 0 additions & 3 deletions
This file was deleted.

0 commit comments

Comments
 (0)