Policy configmap working with kube-scheduler

bindata/v3.11.0/kube-scheduler/policyconfigmap-role.yaml

@@ -0,0 +1,13 @@
+#As of now, system:kube-scheduler role cannot list configmaps from openshift-kube-scheduler namespace. So, creating a role.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: system:openshift:sa-listing-configmaps
+  namespace: openshift-kube-scheduler
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get

bindata/v3.11.0/kube-scheduler/policyconfigmap-rolebinding.yaml

@@ -0,0 +1,12 @@
+# As of now, system:kube-scheduler role cannot list configmaps from openshift-kube-scheduler namespace. So, creating a role.
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  namespace: openshift-kube-scheduler
+  name: system:openshift:sa-listing-configmaps
+roleRef:
+  kind: Role
+  name: system:openshift:sa-listing-configmaps
+subjects:
+- kind: User
+  name: system:kube-scheduler

pkg/operator/resourcesynccontroller/resourcesynccontroller.go

@@ -3,6 +3,8 @@ package resourcesynccontroller
 import (
 	"k8s.io/client-go/kubernetes"
 
+	"github.com/golang/glog"
+	configinformers "github.com/openshift/client-go/config/informers/externalversions"
 	"github.com/openshift/cluster-kube-scheduler-operator/pkg/operator/operatorclient"
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resourcesynccontroller"
@@ -12,6 +14,7 @@ import (
 func NewResourceSyncController(
 	operatorConfigClient v1helpers.OperatorClient,
 	kubeInformersForNamespaces v1helpers.KubeInformersForNamespaces,
+	configInformer configinformers.SharedInformerFactory,
 	kubeClient kubernetes.Interface,
 	eventRecorder events.Recorder) (*resourcesynccontroller.ResourceSyncController, error) {
 
@@ -22,10 +25,17 @@ func NewResourceSyncController(
 		kubeClient.CoreV1(),
 		eventRecorder,
 	)
-	if err := resourceSyncController.SyncConfigMap(
-		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "policy-configmap"},
-		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "policy-configmap"}); err != nil {
-		return nil, err
+
+	scheduler, err := configInformer.Config().V1().Schedulers().Lister().Get("cluster")
+	if err != nil {
+		glog.Infof("Error while listing scheduler %v", err)
+	}
+	if scheduler != nil && len(scheduler.Spec.Policy.Name) > 0 {
+		if err := resourceSyncController.SyncConfigMap(
+			resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: scheduler.Spec.Policy.Name},
+			resourcesynccontroller.ResourceLocation{Namespace: operatorclient.GlobalUserSpecifiedConfigNamespace, Name: "policy-configmap"}); err != nil {
+			return nil, err
+		}
 	}
 	if err := resourceSyncController.SyncSecret(
 		resourcesynccontroller.ResourceLocation{Namespace: operatorclient.TargetNamespace, Name: "kube-scheduler-client-cert-key"},

pkg/operator/starter.go

@@ -13,6 +13,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	configv1 "github.com/openshift/api/config/v1"
+
 	operatorv1 "github.com/openshift/api/operator/v1"
 	configv1client "github.com/openshift/client-go/config/clientset/versioned"
 	configv1informers "github.com/openshift/client-go/config/informers/externalversions"
@@ -79,6 +80,7 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 	resourceSyncController, err := resourcesynccontroller.NewResourceSyncController(
 		operatorClient,
 		kubeInformersForNamespaces,
+		configInformers,
 		kubeClient,
 		ctx.EventRecorder,
 	)
@@ -146,6 +148,7 @@ func RunOperator(ctx *controllercmd.ControllerContext) error {
 	kubeInformersClusterScoped.Start(ctx.Done())
 	kubeInformersNamespace.Start(ctx.Done())
 	kubeInformersForNamespaces.Start(ctx.Done())
+	configInformers.Start(ctx.Done())
 
 	go staticPodControllers.Run(ctx.Done())
 	go resourceSyncController.Run(1, ctx.Done())

pkg/operator/target_config_reconciler.go

@@ -7,7 +7,14 @@ import (
 
 	"github.com/golang/glog"
 
+	operatorv1 "github.com/openshift/api/operator/v1"
 	configinformers "github.com/openshift/client-go/config/informers/externalversions"
+	configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
+	operatorv1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
+	operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
+	"github.com/openshift/cluster-kube-scheduler-operator/pkg/operator/operatorclient"
+	"github.com/openshift/library-go/pkg/operator/events"
+	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -18,25 +25,16 @@ import (
 	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
-
-	operatorv1 "github.com/openshift/api/operator/v1"
-	configlistersv1 "github.com/openshift/client-go/config/listers/config/v1"
-	operatorv1client "github.com/openshift/client-go/operator/clientset/versioned/typed/operator/v1"
-	operatorv1informers "github.com/openshift/client-go/operator/informers/externalversions/operator/v1"
-	"github.com/openshift/cluster-kube-scheduler-operator/pkg/operator/operatorclient"
-	"github.com/openshift/library-go/pkg/operator/events"
-	"github.com/openshift/library-go/pkg/operator/v1helpers"
 )
 
 type TargetConfigReconciler struct {
-	targetImagePullSpec string
-
+	targetImagePullSpec  string
 	operatorConfigClient operatorv1client.KubeSchedulersGetter
-
-	kubeClient       kubernetes.Interface
-	eventRecorder    events.Recorder
-	configMapLister  corev1listers.ConfigMapLister
-	SchedulingLister configlistersv1.SchedulerLister
+	kubeClient           kubernetes.Interface
+	eventRecorder        events.Recorder
+	configMapLister      corev1listers.ConfigMapLister
+	SchedulerLister      configlistersv1.SchedulerLister
+	SchedulingCacheSync  cache.InformerSynced
 	// queue only ever has one item, but it has nice error handling backoff/retry semantics
 	queue workqueue.RateLimitingInterface
 }
@@ -57,10 +55,13 @@ func NewTargetConfigReconciler(
 		kubeClient:           kubeClient,
 		configMapLister:      kubeInformersForNamespaces.ConfigMapLister(),
 		eventRecorder:        eventRecorder,
-		SchedulingLister:     configInformer.Config().V1().Schedulers().Lister(),
+		SchedulerLister:      configInformer.Config().V1().Schedulers().Lister(),
+		SchedulingCacheSync:  configInformer.Config().V1().Schedulers().Informer().HasSynced,
 		queue:                workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "TargetConfigReconciler"),
 	}
 
+	// TODO: @ravig Remove this and move this to config observer code.
+	configInformer.Config().V1().Schedulers().Informer().AddEventHandler(c.eventHandler())
 	operatorConfigInformer.Informer().AddEventHandler(c.eventHandler())
 	namespacedKubeInformers.Rbac().V1().Roles().Informer().AddEventHandler(c.eventHandler())
 	namespacedKubeInformers.Rbac().V1().RoleBindings().Informer().AddEventHandler(c.eventHandler())
@@ -76,8 +77,6 @@ func NewTargetConfigReconciler(
 	// we only watch some namespaces
 	namespacedKubeInformers.Core().V1().Namespaces().Informer().AddEventHandler(c.namespaceEventHandler())
 
-	// TODO: @ravig Remove this and move this to config observer code.
-	configInformer.Config().V1().Schedulers().Informer().AddEventHandler(c.eventHandler())
 	return c
 }
 
@@ -86,7 +85,6 @@ func (c TargetConfigReconciler) sync() error {
 	if err != nil {
 		return err
 	}
-
 	operatorConfigOriginal := operatorConfig.DeepCopy()
 
 	switch operatorConfig.Spec.ManagementState {
@@ -128,6 +126,11 @@ func (c *TargetConfigReconciler) Run(workers int, stopCh <-chan struct{}) {
 	glog.Infof("Starting TargetConfigReconciler")
 	defer glog.Infof("Shutting down TargetConfigReconciler")
 
+	if !cache.WaitForCacheSync(stopCh, c.SchedulingCacheSync) {
+		utilruntime.HandleError(fmt.Errorf("caches did not sync"))
+		return
+	}
+
 	// doesn't matter what workers say, only start one.
 	go wait.Until(c.runWorker, time.Second, stopCh)
 

pkg/operator/target_config_reconciler_v311_00.go

@@ -43,7 +43,7 @@ func createTargetConfigReconciler_v311_00_to_latest(c TargetConfigReconciler, re
 			errors = append(errors, fmt.Errorf("%q (%T): %v", currResult.File, currResult.Type, currResult.Error))
 		}
 	}
-	_, _, err := manageKubeSchedulerConfigMap_v311_00_to_latest(c.configMapLister, c.kubeClient.CoreV1(), recorder, operatorConfig, c.SchedulingLister)
+	_, _, err := manageKubeSchedulerConfigMap_v311_00_to_latest(c.configMapLister, c.kubeClient.CoreV1(), recorder, operatorConfig, c.SchedulerLister)
 	if err != nil {
 		errors = append(errors, fmt.Errorf("%q: %v", "configmap", err))
 	}
@@ -92,6 +92,9 @@ func manageKubeSchedulerConfigMap_v311_00_to_latest(lister corev1listers.ConfigM
 	configMap := resourceread.ReadConfigMapV1OrDie(v311_00_assets.MustAsset("v3.11.0/kube-scheduler/cm.yaml"))
 	var defaultConfig []byte
 	observedpolicyConfigMap, err := schedulerLister.Get("cluster")
+	if err != nil {
+		glog.Infof("Error while listing configmap %v", err.Error())
+	}
 	var policyConfigMapName string
 	if err == nil && observedpolicyConfigMap != nil && len(observedpolicyConfigMap.Spec.Policy.Name) > 0 {
 		policyConfigMapName = observedpolicyConfigMap.Spec.Policy.Name
@@ -105,6 +108,7 @@ func manageKubeSchedulerConfigMap_v311_00_to_latest(lister corev1listers.ConfigM
 			targetPolicyConfigMap.ResourceVersion = ""
 			_, err := client.ConfigMaps(operatorclient.TargetNamespace).Create(targetPolicyConfigMap)
 			if err == nil || apierrors.IsAlreadyExists(err) {
+				glog.Infof("Custom policy config map to be used by scheduler is successfully created")
 				defaultConfig = v311_00_assets.MustAsset("v3.11.0/kube-scheduler/defaultconfig-postbootstrap-with-policy.yaml")
 			} else {
 				// This means policyconfigmap could not be created, so let's default to postbootstrap only.