Implement multicluster client

Tiltfile

@@ -17,7 +17,11 @@ if not os.getenv('TILT_VALUES_PATH'):
     fail("TILT_VALUES_PATH is not set.")
 if not os.path.exists(os.getenv('TILT_VALUES_PATH')):
     fail("TILT_VALUES_PATH "+ os.getenv('TILT_VALUES_PATH') + " does not exist.")
-tilt_values = os.getenv('TILT_VALUES_PATH')
+tilt_values = [os.getenv('TILT_VALUES_PATH')]
+
+tilt_overrides = os.getenv('TILT_OVERRIDES_PATH')
+if tilt_overrides and os.path.exists(tilt_overrides):
+    tilt_values.append(tilt_overrides)
 
 load('ext://helm_resource', 'helm_resource', 'helm_repo')
 helm_repo(
@@ -106,7 +110,7 @@ k8s_yaml(helm('./helm/bundles/cortex-crds', name='cortex-crds', set=[
 
 if 'nova' in ACTIVE_DEPLOYMENTS:
     print("Activating Cortex Nova bundle")
-    k8s_yaml(helm('./helm/bundles/cortex-nova', name='cortex-nova', values=[tilt_values]))
+    k8s_yaml(helm('./helm/bundles/cortex-nova', name='cortex-nova', values=tilt_values))
     k8s_resource('cortex-nova-postgresql', labels=['Cortex-Nova'], port_forwards=[
         port_forward(8000, 5432),
     ])
@@ -125,7 +129,7 @@ if 'nova' in ACTIVE_DEPLOYMENTS:
 
 if 'manila' in ACTIVE_DEPLOYMENTS:
     print("Activating Cortex Manila bundle")
-    k8s_yaml(helm('./helm/bundles/cortex-manila', name='cortex-manila', values=[tilt_values]))
+    k8s_yaml(helm('./helm/bundles/cortex-manila', name='cortex-manila', values=tilt_values))
     k8s_resource('cortex-manila-postgresql', labels=['Cortex-Manila'], port_forwards=[
         port_forward(8002, 5432),
     ])
@@ -142,7 +146,7 @@ if 'manila' in ACTIVE_DEPLOYMENTS:
     )
 
 if 'cinder' in ACTIVE_DEPLOYMENTS:
-    k8s_yaml(helm('./helm/bundles/cortex-cinder', name='cortex-cinder', values=[tilt_values]))
+    k8s_yaml(helm('./helm/bundles/cortex-cinder', name='cortex-cinder', values=tilt_values))
     k8s_resource('cortex-cinder-postgresql', labels=['Cortex-Cinder'], port_forwards=[
         port_forward(8004, 5432),
     ])
@@ -160,7 +164,7 @@ if 'cinder' in ACTIVE_DEPLOYMENTS:
 
 if 'ironcore' in ACTIVE_DEPLOYMENTS:
     print("Activating Cortex IronCore bundle")
-    k8s_yaml(helm('./helm/bundles/cortex-ironcore', name='cortex-ironcore', values=[tilt_values]))
+    k8s_yaml(helm('./helm/bundles/cortex-ironcore', name='cortex-ironcore', values=tilt_values))
     k8s_resource('cortex-ironcore-controller-manager', labels=['Cortex-IronCore'])
     # Deploy resources in machines/samples
     k8s_yaml('samples/ironcore/machinepool.yaml')

api/v1alpha1/datasource_types.go

@@ -293,6 +293,9 @@ type DatasourceList struct {
 	Items           []Datasource `json:"items"`
 }
 
+func (*Datasource) URI() string     { return "datasources.cortex.cloud/v1alpha1" }
+func (*DatasourceList) URI() string { return "datasources.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Datasource{}, &DatasourceList{})
 }

api/v1alpha1/decision_types.go

@@ -155,6 +155,9 @@ type DecisionList struct {
 	Items           []Decision `json:"items"`
 }
 
+func (*Decision) URI() string     { return "decisions.cortex.cloud/v1alpha1" }
+func (*DecisionList) URI() string { return "decisions.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Decision{}, &DecisionList{})
 }

api/v1alpha1/descheduling_types.go

@@ -102,6 +102,9 @@ type DeschedulingList struct {
 	Items           []Descheduling `json:"items"`
 }
 
+func (*Descheduling) URI() string     { return "deschedulings.cortex.cloud/v1alpha1" }
+func (*DeschedulingList) URI() string { return "deschedulings.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Descheduling{}, &DeschedulingList{})
 }

api/v1alpha1/knowledge_types.go

@@ -174,6 +174,9 @@ type KnowledgeList struct {
 	Items           []Knowledge `json:"items"`
 }
 
+func (*Knowledge) URI() string     { return "knowledges.cortex.cloud/v1alpha1" }
+func (*KnowledgeList) URI() string { return "knowledges.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Knowledge{}, &KnowledgeList{})
 }

api/v1alpha1/kpi_types.go

@@ -97,6 +97,9 @@ type KPIList struct {
 	Items           []KPI `json:"items"`
 }
 
+func (*KPI) URI() string     { return "kpis.cortex.cloud/v1alpha1" }
+func (*KPIList) URI() string { return "kpis.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&KPI{}, &KPIList{})
 }

api/v1alpha1/pipeline_types.go

@@ -98,6 +98,9 @@ type PipelineList struct {
 	Items           []Pipeline `json:"items"`
 }
 
+func (*Pipeline) URI() string     { return "pipelines.cortex.cloud/v1alpha1" }
+func (*PipelineList) URI() string { return "pipelines.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Pipeline{}, &PipelineList{})
 }

api/v1alpha1/reservation_types.go

@@ -97,6 +97,9 @@ type ReservationList struct {
 	Items           []Reservation `json:"items"`
 }
 
+func (*Reservation) URI() string     { return "reservations.cortex.cloud/v1alpha1" }
+func (*ReservationList) URI() string { return "reservations.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Reservation{}, &ReservationList{})
 }

api/v1alpha1/step_types.go

@@ -129,6 +129,9 @@ type StepList struct {
 	Items           []Step `json:"items"`
 }
 
+func (*Step) URI() string     { return "steps.cortex.cloud/v1alpha1" }
+func (*StepList) URI() string { return "steps.cortex.cloud/v1alpha1" }
+
 func init() {
 	SchemeBuilder.Register(&Step{}, &StepList{})
 }

cmd/main.go

@@ -22,6 +22,7 @@ import (
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/certwatcher"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/cluster"
 	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
@@ -54,6 +55,7 @@ import (
 	"github.com/cobaltcore-dev/cortex/pkg/conf"
 	"github.com/cobaltcore-dev/cortex/pkg/db"
 	"github.com/cobaltcore-dev/cortex/pkg/monitoring"
+	"github.com/cobaltcore-dev/cortex/pkg/multicluster"
 	"github.com/sapcc/go-bits/httpext"
 	"github.com/sapcc/go-bits/must"
 	corev1 "k8s.io/api/core/v1"
@@ -243,6 +245,33 @@ func main() {
 		os.Exit(1)
 	}
 
+	homeCluster, err := cluster.New(restConfig, func(o *cluster.Options) { o.Scheme = scheme })
+	if err != nil {
+		setupLog.Error(err, "unable to create home cluster")
+		os.Exit(1)
+	}
+	if err := mgr.Add(homeCluster); err != nil {
+		setupLog.Error(err, "unable to add home cluster")
+		os.Exit(1)
+	}
+	multiclusterClient := &multicluster.Client{
+		HomeCluster:    homeCluster,
+		HomeRestConfig: restConfig,
+		HomeScheme:     scheme,
+	}
+	for _, override := range config.APIServerOverrides {
+		cluster, err := multiclusterClient.AddRemote(override.Resource, override.Host, override.CACert)
+		if err != nil {
+			setupLog.Error(err, "unable to create cluster for apiserver override", "apiserver", override.Host)
+			os.Exit(1)
+		}
+		// Also tell the manager about this cluster so that controllers can use it.
+		if err := mgr.Add(cluster); err != nil {
+			setupLog.Error(err, "unable to add cluster for apiserver override", "apiserver", override.Host)
+			os.Exit(1)
+		}
+	}
+
 	// Our custom monitoring registry can add prometheus labels to all metrics.
 	// This is useful to distinguish metrics from different deployments.
 	metrics.Registry = monitoring.WrapRegistry(metrics.Registry, config.Monitoring)
@@ -265,14 +294,14 @@ func main() {
 			Conf:    config,
 		}
 		// Inferred through the base controller.
-		decisionController.Client = mgr.GetClient()
+		decisionController.Client = multiclusterClient
 		decisionController.OperatorName = config.Operator
-		if err := (decisionController).SetupWithManager(mgr); err != nil {
+		if err := (decisionController).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
 			os.Exit(1)
 		}
 		novashims.NewAPI(config, decisionController).Init(mux)
-		go decisionsnova.CleanupNovaDecisionsRegularly(ctx, mgr.GetClient(), config)
+		go decisionsnova.CleanupNovaDecisionsRegularly(ctx, multiclusterClient, config)
 	}
 	if slices.Contains(config.EnabledControllers, "nova-deschedulings-pipeline-controller") {
 		// Deschedulings controller
@@ -284,18 +313,18 @@ func main() {
 			CycleDetector: deschedulingnova.NewCycleDetector(),
 		}
 		// Inferred through the base controller.
-		deschedulingsController.Client = mgr.GetClient()
+		deschedulingsController.Client = multiclusterClient
 		deschedulingsController.OperatorName = config.Operator
-		if err := (deschedulingsController).SetupWithManager(mgr); err != nil {
+		if err := (deschedulingsController).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DeschedulingsReconciler")
 			os.Exit(1)
 		}
 		go deschedulingsController.CreateDeschedulingsPeriodically(ctx)
 		// Deschedulings cleanup on startup
 		if err := (&deschedulingnova.Cleanup{
-			Client: mgr.GetClient(),
+			Client: multiclusterClient,
 			Scheme: mgr.GetScheme(),
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "Cleanup")
 			os.Exit(1)
 		}
@@ -306,39 +335,39 @@ func main() {
 			Conf:    config,
 		}
 		// Inferred through the base controller.
-		controller.Client = mgr.GetClient()
+		controller.Client = multiclusterClient
 		controller.OperatorName = config.Operator
-		if err := (controller).SetupWithManager(mgr); err != nil {
+		if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
 			os.Exit(1)
 		}
 		manilashims.NewAPI(config, controller).Init(mux)
-		go decisionsmanila.CleanupManilaDecisionsRegularly(ctx, mgr.GetClient(), config)
+		go decisionsmanila.CleanupManilaDecisionsRegularly(ctx, multiclusterClient, config)
 	}
 	if slices.Contains(config.EnabledControllers, "cinder-decisions-pipeline-controller") {
 		controller := &decisionscinder.DecisionPipelineController{
 			Monitor: pipelineMonitor,
 			Conf:    config,
 		}
 		// Inferred through the base controller.
-		controller.Client = mgr.GetClient()
+		controller.Client = multiclusterClient
 		controller.OperatorName = config.Operator
-		if err := (controller).SetupWithManager(mgr); err != nil {
+		if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
 			os.Exit(1)
 		}
 		cindershims.NewAPI(config, controller).Init(mux)
-		go decisionscinder.CleanupCinderDecisionsRegularly(ctx, mgr.GetClient(), config)
+		go decisionscinder.CleanupCinderDecisionsRegularly(ctx, multiclusterClient, config)
 	}
 	if slices.Contains(config.EnabledControllers, "ironcore-decisions-pipeline-controller") {
 		controller := &decisionsmachines.DecisionPipelineController{
 			Monitor: pipelineMonitor,
 			Conf:    config,
 		}
 		// Inferred through the base controller.
-		controller.Client = mgr.GetClient()
+		controller.Client = multiclusterClient
 		controller.OperatorName = config.Operator
-		if err := (controller).SetupWithManager(mgr); err != nil {
+		if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
 			os.Exit(1)
 		}
@@ -347,23 +376,23 @@ func main() {
 		// Setup a controller which will reconcile the history and explanation for
 		// decision resources.
 		explanationController := &explanation.Controller{
-			Client:       mgr.GetClient(),
+			Client:       multiclusterClient,
 			OperatorName: config.Operator,
 		}
-		if err := explanationController.SetupWithManager(mgr); err != nil {
+		if err := explanationController.SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "ExplanationController")
 			os.Exit(1)
 		}
 	}
 	if slices.Contains(config.EnabledControllers, "reservations-controller") {
-		monitor := reservationscontroller.NewControllerMonitor(mgr.GetClient())
+		monitor := reservationscontroller.NewControllerMonitor(multiclusterClient)
 		metrics.Registry.MustRegister(&monitor)
 		if err := (&reservationscontroller.ReservationReconciler{
-			Client:           mgr.GetClient(),
+			Client:           multiclusterClient,
 			Scheme:           mgr.GetScheme(),
 			Conf:             config,
 			HypervisorClient: reservationscontroller.NewHypervisorClient(),
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "Reservation")
 			os.Exit(1)
 		}
@@ -372,20 +401,20 @@ func main() {
 		monitor := datasources.NewMonitor()
 		metrics.Registry.MustRegister(&monitor)
 		if err := (&openstack.OpenStackDatasourceReconciler{
-			Client:  mgr.GetClient(),
+			Client:  multiclusterClient,
 			Scheme:  mgr.GetScheme(),
 			Monitor: monitor,
 			Conf:    config,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "OpenStackDatasourceReconciler")
 			os.Exit(1)
 		}
 		if err := (&prometheus.PrometheusDatasourceReconciler{
-			Client:  mgr.GetClient(),
+			Client:  multiclusterClient,
 			Scheme:  mgr.GetScheme(),
 			Monitor: monitor,
 			Conf:    config,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "PrometheusDatasourceReconciler")
 			os.Exit(1)
 		}
@@ -394,29 +423,29 @@ func main() {
 		monitor := extractor.NewMonitor()
 		metrics.Registry.MustRegister(&monitor)
 		if err := (&extractor.KnowledgeReconciler{
-			Client:  mgr.GetClient(),
+			Client:  multiclusterClient,
 			Scheme:  mgr.GetScheme(),
 			Monitor: monitor,
 			Conf:    config,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "KnowledgeReconciler")
 			os.Exit(1)
 		}
 		if err := (&extractor.TriggerReconciler{
-			Client: mgr.GetClient(),
+			Client: multiclusterClient,
 			Scheme: mgr.GetScheme(),
 			Conf:   config,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "TriggerReconciler")
 			os.Exit(1)
 		}
 	}
 	if slices.Contains(config.EnabledControllers, "kpis-controller") {
 		if err := (&kpis.Controller{
-			Client:              mgr.GetClient(),
+			Client:              multiclusterClient,
 			SupportedKPIsByImpl: kpis.SupportedKPIsByImpl,
 			OperatorName:        config.Operator,
-		}).SetupWithManager(mgr); err != nil {
+		}).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "KPIController")
 			os.Exit(1)
 		}

docs/guides/multicluster/cortex-home-crb.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: grant-cortex-remote-oidc-access
+subjects:
+- kind: User
+  apiGroup: rbac.authorization.k8s.io
+  name: system:anonymous
+roleRef:
+  kind: ClusterRole
+  name: system:service-account-issuer-discovery
+  apiGroup: rbac.authorization.k8s.io

docs/guides/multicluster/cortex-home.yaml

@@ -0,0 +1,23 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+name: cortex-home
+nodes:
+  - role: worker
+  - role: control-plane
+    extraPortMappings:
+    - containerPort: 6443
+      hostPort: 8443
+    kubeadmConfigPatches:
+      - |
+        kind: ClusterConfiguration
+        apiServer:
+          extraArgs:
+            service-account-issuer: "https://host.docker.internal:8443"
+            service-account-jwks-uri: "https://host.docker.internal:8443/openid/v1/jwks"
+          certSANs:
+            - api-proxy
+            - api-proxy.default.svc
+            - api-proxy.default.svc.cluster.local
+            - localhost
+            - 127.0.0.1
+            - host.docker.internal