Merge pull request #265 from kerthcet/feat/add-internal-cert

Feat: add internal cert to replace cert manager

config/default/kustomization.yaml

@@ -20,7 +20,8 @@ bases:
 # crd/kustomization.yaml
 - ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
-- ../certmanager
+# - ../certmanager
+- ../internalcert
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 
@@ -41,34 +42,34 @@ patchesStrategicMerge:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
 # Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
 # 'CERTMANAGER' needs to be enabled to use ca injection
-- webhookcainjection_patch.yaml
+# - webhookcainjection_patch.yaml
 
 # the following config is for teaching kustomize how to do var substitution
-vars:
+# vars:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
-- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-  fieldref:
-    fieldpath: metadata.namespace
-- name: CERTIFICATE_NAME
-  objref:
-    kind: Certificate
-    group: cert-manager.io
-    version: v1
-    name: serving-cert # this name should match the one in certificate.yaml
-- name: SERVICE_NAMESPACE # namespace of the service
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service
-  fieldref:
-    fieldpath: metadata.namespace
-- name: SERVICE_NAME
-  objref:
-    kind: Service
-    version: v1
-    name: webhook-service
+# - name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
+#   objref:
+#     kind: Certificate
+#     group: cert-manager.io
+#     version: v1
+#     name: serving-cert # this name should match the one in certificate.yaml
+#   fieldref:
+#     fieldpath: metadata.namespace
+# - name: CERTIFICATE_NAME
+#   objref:
+#     kind: Certificate
+#     group: cert-manager.io
+#     version: v1
+#     name: serving-cert # this name should match the one in certificate.yaml
+# - name: SERVICE_NAMESPACE # namespace of the service
+#   objref:
+#     kind: Service
+#     version: v1
+#     name: webhook-service
+#   fieldref:
+#     fieldpath: metadata.namespace
+# - name: SERVICE_NAME
+#   objref:
+#     kind: Service
+#     version: v1
+#     name: webhook-service

config/internalcert/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+- secret.yaml

config/internalcert/secret.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: webhook-server-cert
+  namespace: system

config/rbac/role.yaml

@@ -21,6 +21,33 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - update
+  - watch
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - validatingwebhookconfigurations
+  verbs:
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - batch
   resources:

docs/setup/install.md

@@ -7,7 +7,6 @@ Make sure the following conditions are met:
 - A Kubernetes cluster with version 1.21 or newer is running. Learn how to [install the Kubernetes tools](https://kubernetes.io/docs/tasks/tools/).
 - The SuspendJob [feature gate](https://kubernetes.io/docs/reference/command-line-tools-reference/feature-gates/) is enabled. In Kubernetes 1.22 or newer, the feature gate is enabled by default.
 - The kubectl command-line tool has communication with your cluster.
-- The [cert-manager](https://github.com/cert-manager/cert-manager) is installed in your cluster. Learn how to [install cert-manager](https://cert-manager.io/docs/installation/).
 
 ## Install a released version
 
@@ -20,7 +19,7 @@ kubectl apply -f https://github.com/kubernetes-sigs/kueue/releases/download/$VER
 
 ## Install a custom-configured released version
 
-To install a custom-configured released version of Kueue in your cluster, execute the following steps: 
+To install a custom-configured released version of Kueue in your cluster, execute the following steps:
 
 1. Download the release's `manifests.yaml` file:
 
@@ -29,9 +28,9 @@ VERSION=v0.1.0
 wget https://github.com/kubernetes-sigs/kueue/releases/download/$VERSION/manifests.yaml
 ```
 2. With an editor of your preference, open `manifests.yaml`.
-3. In the `kueue-manager-config` ConfigMap manifest, edit the 
+3. In the `kueue-manager-config` ConfigMap manifest, edit the
 `controller_manager_config.yaml` data entry. The entry represents
-the default Kueue Configuration 
+the default Kueue Configuration
 struct ([v1alpha1@v0.1.0](https://pkg.go.dev/sigs.k8s.io/kueue@v0.1.0/apis/config/v1alpha1#Configuration)).
 The contents of the ConfigMap are similar to the following:
 

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/google/go-cmp v0.5.7
 	github.com/onsi/ginkgo/v2 v2.1.4
 	github.com/onsi/gomega v1.19.0
+	github.com/open-policy-agent/cert-controller v0.3.0
 	github.com/prometheus/client_golang v1.12.1
 	go.uber.org/zap v1.21.0
 	k8s.io/api v0.23.4

go.sum

@@ -396,6 +396,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw=
 github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro=
+github.com/open-policy-agent/cert-controller v0.3.0 h1:9eUgN3yYMZsfyW7qdW8+CX9YZCUb5R5JfRTj0cqaSVg=
+github.com/open-policy-agent/cert-controller v0.3.0/go.mod h1:uOQW+2tMU51vSxy1Yt162oVUTMdqLuotC0aObQxrh6k=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -1034,6 +1036,7 @@ k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
 k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
 k8s.io/klog/v2 v2.40.1 h1:P4RRucWk/lFOlDdkAr3mc7iWFkgKrZY9qZMAgek06S4=
 k8s.io/klog/v2 v2.40.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0=
+k8s.io/kube-aggregator v0.23.2 h1:6CoZZqNdFc9benrgSJJ0GQGgFtKjI0y3UwlBbioXtc8=
 k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=
 k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf h1:M9XBsiMslw2lb2ZzglC0TOkBPK5NQi0/noUrdnoFwUg=
 k8s.io/kube-openapi v0.0.0-20220124234850-424119656bbf/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk=

main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"bytes"
+	"context"
 	"flag"
 	"fmt"
 	"os"
@@ -46,6 +47,7 @@ import (
 	"sigs.k8s.io/kueue/pkg/metrics"
 	"sigs.k8s.io/kueue/pkg/queue"
 	"sigs.k8s.io/kueue/pkg/scheduler"
+	"sigs.k8s.io/kueue/pkg/util/cert"
 	//+kubebuilder:scaffold:imports
 )
 
@@ -108,31 +110,79 @@ func main() {
 		setupLog.Error(err, "unable to start manager")
 		os.Exit(1)
 	}
+
+	certsReady := make(chan struct{})
+	if err = cert.ManageCerts(mgr, certsReady); err != nil {
+		setupLog.Error(err, "unable to set up cert rotation")
+		os.Exit(1)
+	}
+
+	cCache := cache.New(mgr.GetClient())
+	queues := queue.NewManager(mgr.GetClient(), cCache)
+
+	setupIndexes(mgr)
+
+	setupProbeEndpoints(mgr)
+	// Cert won't be ready until manager starts, so start a goroutine here which
+	// will block until the cert is ready before setting up the controllers.
+	// Controllers who register after manager starts will start directly.
+	go setupControllers(mgr, cCache, queues, certsReady, config.ManageJobsWithoutQueueName)
+
+	ctx := ctrl.SetupSignalHandler()
+	go func() {
+		queues.CleanUpOnContext(ctx)
+	}()
+
+	setupScheduler(ctx, mgr, cCache, queues)
+
+	setupLog.Info("starting manager")
+	if err := mgr.Start(ctx); err != nil {
+		setupLog.Error(err, "problem running manager")
+		os.Exit(1)
+	}
+}
+
+func setupIndexes(mgr ctrl.Manager) {
 	if err := queue.SetupIndexes(mgr.GetFieldIndexer()); err != nil {
 		setupLog.Error(err, "Unable to setup queue indexes")
 	}
 	if err := cache.SetupIndexes(mgr.GetFieldIndexer()); err != nil {
 		setupLog.Error(err, "Unable to setup cache indexes")
 	}
+	if err := job.SetupIndexes(mgr.GetFieldIndexer()); err != nil {
+		setupLog.Error(err, "Unable to setup job indexes")
+	}
+}
+
+func setupControllers(mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager, certsReady chan struct{}, manageJobsWithoutQueueName bool) {
+	// The controllers won't work until the webhooks are operating, and the webhook won't work until the
+	// certs are all in place.
+	setupLog.Info("Waiting for certificate generation to complete")
+	<-certsReady
+	setupLog.Info("Certs ready")
 
-	cCache := cache.New(mgr.GetClient())
-	queues := queue.NewManager(mgr.GetClient(), cCache)
 	if failedCtrl, err := core.SetupControllers(mgr, queues, cCache); err != nil {
 		setupLog.Error(err, "Unable to create controller", "controller", failedCtrl)
+		os.Exit(1)
 	}
-	if err = job.NewReconciler(mgr.GetScheme(),
+	if err := job.NewReconciler(mgr.GetScheme(),
 		mgr.GetClient(),
 		mgr.GetEventRecorderFor(constants.JobControllerName),
-		job.WithManageJobsWithoutQueueName(config.ManageJobsWithoutQueueName),
+		job.WithManageJobsWithoutQueueName(manageJobsWithoutQueueName),
 	).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Job")
 		os.Exit(1)
 	}
-	if err = (&kueuev1alpha1.Workload{}).SetupWebhookWithManager(mgr); err != nil {
+	if err := (&kueuev1alpha1.Workload{}).SetupWebhookWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create webhook", "webhook", "Workload")
 		os.Exit(1)
 	}
 	//+kubebuilder:scaffold:builder
+}
+
+// setupProbeEndpoints registers the health endpoints
+func setupProbeEndpoints(mgr ctrl.Manager) {
+	defer setupLog.Info("Probe endpoints are configured on healthz and readyz")
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {
 		setupLog.Error(err, "unable to set up health check")
@@ -142,21 +192,16 @@ func main() {
 		setupLog.Error(err, "unable to set up ready check")
 		os.Exit(1)
 	}
+}
 
-	ctx := ctrl.SetupSignalHandler()
-	go func() {
-		queues.CleanUpOnContext(ctx)
-	}()
-	sched := scheduler.New(queues, cCache, mgr.GetClient(),
-		mgr.GetEventRecorderFor(constants.ManagerName))
-	go func() {
-		sched.Start(ctx)
-	}()
-	setupLog.Info("starting manager")
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		os.Exit(1)
-	}
+func setupScheduler(ctx context.Context, mgr ctrl.Manager, cCache *cache.Cache, queues *queue.Manager) {
+	sched := scheduler.New(
+		queues,
+		cCache,
+		mgr.GetClient(),
+		mgr.GetEventRecorderFor(constants.ManagerName),
+	)
+	go sched.Start(ctx)
 }
 
 func encodeConfig(cfg *configv1alpha1.Configuration) (string, error) {

pkg/controller/workload/job/job_controller.go

@@ -91,9 +91,16 @@ func NewReconciler(
 // SetupWithManager sets up the controller with the Manager. It indexes workloads
 // based on the owning jobs.
 func (r *JobReconciler) SetupWithManager(mgr ctrl.Manager) error {
-	if err := mgr.GetFieldIndexer().IndexField(context.Background(), &kueue.Workload{}, ownerKey, func(rawObj client.Object) []string {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&batchv1.Job{}).
+		Owns(&kueue.Workload{}).
+		Complete(r)
+}
+
+func SetupIndexes(indexer client.FieldIndexer) error {
+	return indexer.IndexField(context.Background(), &kueue.Workload{}, ownerKey, func(o client.Object) []string {
 		// grab the Workload object, extract the owner...
-		wl := rawObj.(*kueue.Workload)
+		wl := o.(*kueue.Workload)
 		owner := metav1.GetControllerOf(wl)
 		if owner == nil {
 			return nil
@@ -102,17 +109,9 @@ func (r *JobReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		if owner.APIVersion != "batch/v1" || owner.Kind != "Job" {
 			return nil
 		}
-
 		// ...and if so, return it
 		return []string{owner.Name}
-	}); err != nil {
-		return err
-	}
-
-	return ctrl.NewControllerManagedBy(mgr).
-		For(&batchv1.Job{}).
-		Owns(&kueue.Workload{}).
-		Complete(r)
+	})
 }
 
 //+kubebuilder:rbac:groups=scheduling.k8s.io,resources=priorityclasses,verbs=list;get;watch