diff --git a/bindata/v3.11.0/kube-scheduler/policyconfigmap-role.yaml b/bindata/v3.11.0/kube-scheduler/policyconfigmap-role.yaml new file mode 100644 index 000000000..095b77256 --- /dev/null +++ b/bindata/v3.11.0/kube-scheduler/policyconfigmap-role.yaml @@ -0,0 +1,13 @@ +#As of now, system:kube-scheduler role cannot list configmaps from openshift-kube-scheduler namespace. So, creating a role. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system:openshift:sa-listing-configmaps + namespace: openshift-kube-scheduler +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get diff --git a/bindata/v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml b/bindata/v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml new file mode 100644 index 000000000..e23757cfc --- /dev/null +++ b/bindata/v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml @@ -0,0 +1,12 @@ +# As of now, system:kube-scheduler role cannot list configmaps from openshift-kube-scheduler namespace. So, creating a role. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: openshift-kube-scheduler + name: system:openshift:sa-listing-configmaps +roleRef: + kind: Role + name: system:openshift:sa-listing-configmaps +subjects: +- kind: User + name: system:kube-scheduler diff --git a/pkg/operator/resourcesynccontroller/resourcesynccontroller.go b/pkg/operator/resourcesynccontroller/resourcesynccontroller.go index aa46df56d..1c0d6d30b 100644 --- a/pkg/operator/resourcesynccontroller/resourcesynccontroller.go +++ b/pkg/operator/resourcesynccontroller/resourcesynccontroller.go @@ -3,6 +3,8 @@ package resourcesynccontroller import ( "k8s.io/client-go/kubernetes" + "github.com/golang/glog" + configinformers "github.com/openshift/client-go/config/informers/externalversions" "github.com/openshift/cluster-kube-scheduler-operator/pkg/operator/operatorclient" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resourcesynccontroller" @@ -12,6 +14,7 @@ import ( func NewResourceSyncController( operatorConfigClient v1helpers.OperatorClient, kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces, + configInformer configinformers.SharedInformerFactory, kubeClient kubernetes.Interface, eventRecorder events.Recorder) (*resourcesynccontroller.ResourceSyncController, error) { @@ -22,10 +25,17 @@ func NewResourceSyncController( kubeClient.CoreV1(), eventRecorder, ) - if err := resourceSyncController.SyncConfigMap( - resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "policy-configmap"}, - resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "policy-configmap"}); err != nil { - return nil, err + + scheduler, err := configInformer.Config().V1().Schedulers().Lister().Get("cluster") + if err != nil { + glog.Infof("Error while listing scheduler %v", err) + } + if scheduler != nil && len(scheduler.Spec.Policy.Name) > 0 { + if err := resourceSyncController.SyncConfigMap( + resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: scheduler.Spec.Policy.Name}, + resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "policy-configmap"}); err != nil { + return nil, err + } } if err := resourceSyncController.SyncSecret( resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "kube-scheduler-client-cert-key"}, diff --git a/pkg/operator/starter.go b/pkg/operator/starter.go index 6e8886756..52913165f 100644 --- a/pkg/operator/starter.go +++ b/pkg/operator/starter.go @@ -13,6 +13,7 @@ import ( "k8s.io/client-go/kubernetes" configv1 "github.com/openshift/api/config/v1" + operatorv1 "github.com/openshift/api/operator/v1" configv1client "github.com/openshift/client-go/config/clientset/versioned" configv1informers "github.com/openshift/client-go/config/informers/externalversions" @@ -79,6 +80,7 @@ func RunOperator(ctx *controllercmd.ControllerContext) error { resourceSyncController, err := resourcesynccontroller.NewResourceSyncController( operatorClient, kubeInformersForNamespaces, + configInformers, kubeClient, ctx.EventRecorder, ) @@ -146,6 +148,7 @@ func RunOperator(ctx *controllercmd.ControllerContext) error { kubeInformersClusterScoped.Start(ctx.Done()) kubeInformersNamespace.Start(ctx.Done()) kubeInformersForNamespaces.Start(ctx.Done()) + configInformers.Start(ctx.Done()) go staticPodControllers.Run(ctx.Done()) go resourceSyncController.Run(1, ctx.Done()) diff --git a/pkg/operator/target_config_reconciler.go b/pkg/operator/target_config_reconciler.go index 62175b05e..5c676e107 100644 --- a/pkg/operator/target_config_reconciler.go +++ b/pkg/operator/target_config_reconciler.go @@ -7,7 +7,14 @@ import ( "github.com/golang/glog" + operatorv1 "github.com/openshift/api/operator/v1" configinformers "github.com/openshift/client-go/config/informers/externalversions" + configlistersv1 "github.com/openshift/client-go/config/listers/config/v1" + operatorv1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1" + operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" + "github.com/openshift/cluster-kube-scheduler-operator/pkg/operator/operatorclient" + "github.com/openshift/library-go/pkg/operator/events" + "github.com/openshift/library-go/pkg/operator/v1helpers" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" @@ -18,25 +25,16 @@ import ( corev1listers "k8s.io/client-go/listers/core/v1" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - - operatorv1 "github.com/openshift/api/operator/v1" - configlistersv1 "github.com/openshift/client-go/config/listers/config/v1" - operatorv1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1" - operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1" - "github.com/openshift/cluster-kube-scheduler-operator/pkg/operator/operatorclient" - "github.com/openshift/library-go/pkg/operator/events" - "github.com/openshift/library-go/pkg/operator/v1helpers" ) type TargetConfigReconciler struct { - targetImagePullSpec string - + targetImagePullSpec string operatorConfigClient operatorv1client.KubeSchedulersGetter - - kubeClient kubernetes.Interface - eventRecorder events.Recorder - configMapLister corev1listers.ConfigMapLister - SchedulingLister configlistersv1.SchedulerLister + kubeClient kubernetes.Interface + eventRecorder events.Recorder + configMapLister corev1listers.ConfigMapLister + SchedulerLister configlistersv1.SchedulerLister + SchedulingCacheSync cache.InformerSynced // queue only ever has one item, but it has nice error handling backoff/retry semantics queue workqueue.RateLimitingInterface } @@ -57,10 +55,13 @@ func NewTargetConfigReconciler( kubeClient: kubeClient, configMapLister: kubeInformersForNamespaces.ConfigMapLister(), eventRecorder: eventRecorder, - SchedulingLister: configInformer.Config().V1().Schedulers().Lister(), + SchedulerLister: configInformer.Config().V1().Schedulers().Lister(), + SchedulingCacheSync: configInformer.Config().V1().Schedulers().Informer().HasSynced, queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "TargetConfigReconciler"), } + // TODO: @ravig Remove this and move this to config observer code. + configInformer.Config().V1().Schedulers().Informer().AddEventHandler(c.eventHandler()) operatorConfigInformer.Informer().AddEventHandler(c.eventHandler()) namespacedKubeInformers.Rbac().V1().Roles().Informer().AddEventHandler(c.eventHandler()) namespacedKubeInformers.Rbac().V1().RoleBindings().Informer().AddEventHandler(c.eventHandler()) @@ -76,8 +77,6 @@ func NewTargetConfigReconciler( // we only watch some namespaces namespacedKubeInformers.Core().V1().Namespaces().Informer().AddEventHandler(c.namespaceEventHandler()) - // TODO: @ravig Remove this and move this to config observer code. - configInformer.Config().V1().Schedulers().Informer().AddEventHandler(c.eventHandler()) return c } @@ -86,7 +85,6 @@ func (c TargetConfigReconciler) sync() error { if err != nil { return err } - operatorConfigOriginal := operatorConfig.DeepCopy() switch operatorConfig.Spec.ManagementState { @@ -128,6 +126,11 @@ func (c *TargetConfigReconciler) Run(workers int, stopCh <-chan struct{}) { glog.Infof("Starting TargetConfigReconciler") defer glog.Infof("Shutting down TargetConfigReconciler") + if !cache.WaitForCacheSync(stopCh, c.SchedulingCacheSync) { + utilruntime.HandleError(fmt.Errorf("caches did not sync")) + return + } + // doesn't matter what workers say, only start one. go wait.Until(c.runWorker, time.Second, stopCh) diff --git a/pkg/operator/target_config_reconciler_v311_00.go b/pkg/operator/target_config_reconciler_v311_00.go index 782901a69..0255ad645 100644 --- a/pkg/operator/target_config_reconciler_v311_00.go +++ b/pkg/operator/target_config_reconciler_v311_00.go @@ -43,7 +43,7 @@ func createTargetConfigReconciler_v311_00_to_latest(c TargetConfigReconciler, re errors = append(errors, fmt.Errorf("%q (%T): %v", currResult.File, currResult.Type, currResult.Error)) } } - _, _, err := manageKubeSchedulerConfigMap_v311_00_to_latest(c.configMapLister, c.kubeClient.CoreV1(), recorder, operatorConfig, c.SchedulingLister) + _, _, err := manageKubeSchedulerConfigMap_v311_00_to_latest(c.configMapLister, c.kubeClient.CoreV1(), recorder, operatorConfig, c.SchedulerLister) if err != nil { errors = append(errors, fmt.Errorf("%q: %v", "configmap", err)) } @@ -92,6 +92,9 @@ func manageKubeSchedulerConfigMap_v311_00_to_latest(lister corev1listers.ConfigM configMap := resourceread.ReadConfigMapV1OrDie(v311_00_assets.MustAsset("v3.11.0/kube-scheduler/cm.yaml")) var defaultConfig []byte observedpolicyConfigMap, err := schedulerLister.Get("cluster") + if err != nil { + glog.Infof("Error while listing configmap %v", err.Error()) + } var policyConfigMapName string if err == nil && observedpolicyConfigMap != nil && len(observedpolicyConfigMap.Spec.Policy.Name) > 0 { policyConfigMapName = observedpolicyConfigMap.Spec.Policy.Name @@ -105,6 +108,7 @@ func manageKubeSchedulerConfigMap_v311_00_to_latest(lister corev1listers.ConfigM targetPolicyConfigMap.ResourceVersion = "" _, err := client.ConfigMaps(operatorclient.TargetNamespace).Create(targetPolicyConfigMap) if err == nil || apierrors.IsAlreadyExists(err) { + glog.Infof("Custom policy config map to be used by scheduler is successfully created") defaultConfig = v311_00_assets.MustAsset("v3.11.0/kube-scheduler/defaultconfig-postbootstrap-with-policy.yaml") } else { // This means policyconfigmap could not be created, so let's default to postbootstrap only. diff --git a/pkg/operator/v311_00_assets/bindata.go b/pkg/operator/v311_00_assets/bindata.go index 65a9a82fe..3b4df8750 100644 --- a/pkg/operator/v311_00_assets/bindata.go +++ b/pkg/operator/v311_00_assets/bindata.go @@ -10,6 +10,8 @@ // bindata/v3.11.0/kube-scheduler/operator-config.yaml // bindata/v3.11.0/kube-scheduler/pod-cm.yaml // bindata/v3.11.0/kube-scheduler/pod.yaml +// bindata/v3.11.0/kube-scheduler/policyconfigmap-role.yaml +// bindata/v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml // bindata/v3.11.0/kube-scheduler/sa.yaml // bindata/v3.11.0/kube-scheduler/scheduler-clusterrolebinding.yaml // bindata/v3.11.0/kube-scheduler/svc.yaml @@ -344,6 +346,66 @@ func v3110KubeSchedulerPodYaml() (*asset, error) { return a, nil } +var _v3110KubeSchedulerPolicyconfigmapRoleYaml = []byte(`#As of now, system:kube-scheduler role cannot list configmaps from openshift-kube-scheduler namespace. So, creating a role. +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: system:openshift:sa-listing-configmaps + namespace: openshift-kube-scheduler +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get +`) + +func v3110KubeSchedulerPolicyconfigmapRoleYamlBytes() ([]byte, error) { + return _v3110KubeSchedulerPolicyconfigmapRoleYaml, nil +} + +func v3110KubeSchedulerPolicyconfigmapRoleYaml() (*asset, error) { + bytes, err := v3110KubeSchedulerPolicyconfigmapRoleYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "v3.11.0/kube-scheduler/policyconfigmap-role.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + +var _v3110KubeSchedulerPolicyconfigmapRolebindingYaml = []byte(`# As of now, system:kube-scheduler role cannot list configmaps from openshift-kube-scheduler namespace. So, creating a role. +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + namespace: openshift-kube-scheduler + name: system:openshift:sa-listing-configmaps +roleRef: + kind: Role + name: system:openshift:sa-listing-configmaps +subjects: +- kind: ServiceAccount + namespace: openshift-kube-scheduler + name: openshift-kube-scheduler-sa +`) + +func v3110KubeSchedulerPolicyconfigmapRolebindingYamlBytes() ([]byte, error) { + return _v3110KubeSchedulerPolicyconfigmapRolebindingYaml, nil +} + +func v3110KubeSchedulerPolicyconfigmapRolebindingYaml() (*asset, error) { + bytes, err := v3110KubeSchedulerPolicyconfigmapRolebindingYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _v3110KubeSchedulerSaYaml = []byte(`apiVersion: v1 kind: ServiceAccount metadata: @@ -490,6 +552,8 @@ var _bindata = map[string]func() (*asset, error){ "v3.11.0/kube-scheduler/operator-config.yaml": v3110KubeSchedulerOperatorConfigYaml, "v3.11.0/kube-scheduler/pod-cm.yaml": v3110KubeSchedulerPodCmYaml, "v3.11.0/kube-scheduler/pod.yaml": v3110KubeSchedulerPodYaml, + "v3.11.0/kube-scheduler/policyconfigmap-role.yaml": v3110KubeSchedulerPolicyconfigmapRoleYaml, + "v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml": v3110KubeSchedulerPolicyconfigmapRolebindingYaml, "v3.11.0/kube-scheduler/sa.yaml": v3110KubeSchedulerSaYaml, "v3.11.0/kube-scheduler/scheduler-clusterrolebinding.yaml": v3110KubeSchedulerSchedulerClusterrolebindingYaml, "v3.11.0/kube-scheduler/svc.yaml": v3110KubeSchedulerSvcYaml, @@ -548,6 +612,8 @@ var _bintree = &bintree{nil, map[string]*bintree{ "operator-config.yaml": {v3110KubeSchedulerOperatorConfigYaml, map[string]*bintree{}}, "pod-cm.yaml": {v3110KubeSchedulerPodCmYaml, map[string]*bintree{}}, "pod.yaml": {v3110KubeSchedulerPodYaml, map[string]*bintree{}}, + "policyconfigmap-role.yaml": {v3110KubeSchedulerPolicyconfigmapRoleYaml, map[string]*bintree{}}, + "policyconfigmap-rolebinding.yaml": {v3110KubeSchedulerPolicyconfigmapRolebindingYaml, map[string]*bintree{}}, "sa.yaml": {v3110KubeSchedulerSaYaml, map[string]*bintree{}}, "scheduler-clusterrolebinding.yaml": {v3110KubeSchedulerSchedulerClusterrolebindingYaml, map[string]*bintree{}}, "svc.yaml": {v3110KubeSchedulerSvcYaml, map[string]*bintree{}},