diff --git a/Dockerfile b/Dockerfile index 3006998..3fefe4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,17 @@ FROM registry.access.redhat.com/ubi8/go-toolset:1.19.10-10 as builder WORKDIR /workspace -# Copy the Go Modules manifests +# Copy the Go modules manifests COPY go.mod go.mod COPY go.sum go.sum # cache deps before building and copying source so that we don't need to re-download as much # and so that source changes don't invalidate our downloaded layer RUN go mod download -# Copy the go source +# Copy the Go sources COPY main.go main.go COPY controllers/ controllers/ +COPY pkg/ pkg/ # Build USER root diff --git a/controllers/appwrapper_controller.go b/controllers/appwrapper_controller.go index 67a09bb..0f5010e 100644 --- a/controllers/appwrapper_controller.go +++ b/controllers/appwrapper_controller.go @@ -18,21 +18,22 @@ package controllers import ( "context" + "fmt" "os" - "strconv" "strings" "time" mapiclientset "github.com/openshift/client-go/machine/clientset/versioned" machineinformersv1beta1 "github.com/openshift/client-go/machine/informers/externalversions" "github.com/openshift/client-go/machine/listers/machine/v1beta1" - - appwrapperClientSet "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" + "github.com/project-codeflare/instascale/pkg/config" arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + appwrapperClientSet "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/clientset/versioned" + arbinformersFactory "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/informers/externalversions" appwrapperlisters "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/listers/controller/v1beta1" - arbinformersFactory "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/client/informers/externalversions" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -40,6 +41,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/klog" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/log" @@ -48,9 +50,8 @@ import ( // AppWrapperReconciler reconciles a AppWrapper object type AppWrapperReconciler struct { client.Client - Scheme *runtime.Scheme - ConfigsNamespace string - OcmSecretNamespace string + Scheme *runtime.Scheme + Config config.InstaScaleConfiguration } var ( @@ -137,7 +138,6 @@ func (r *AppWrapperReconciler) Reconcile(ctx context.Context, req ctrl.Request) // SetupWithManager sets up the controller with the Manager. func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error { - kubeconfig := os.Getenv("KUBECONFIG") cb, err := NewClientBuilder(kubeconfig) if err != nil { @@ -150,31 +150,23 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error { machineClient = cb.MachineClientOrDie("machine-shared-informer") kubeClient, _ = kubernetes.NewForConfig(restConfig) - instascaleConfigmap, err := kubeClient.CoreV1().ConfigMaps(r.ConfigsNamespace).Get(context.Background(), "instascale-config", metav1.GetOptions{}) - if err != nil { - klog.Infof("Config map named instascale-config is not available in namespace %v", r.ConfigsNamespace) - return err - } - maxScaleNodesAllowed, err = strconv.Atoi(instascaleConfigmap.Data["maxScaleoutAllowed"]) - if err != nil { - klog.Warningf("Error converting %v to int. Setting maxScaleNodesAllowed to 3", maxScaleNodesAllowed) - maxScaleNodesAllowed = 3 - } - if instascaleConfigmap != nil { - klog.Errorf("Got config map named %v from namespace %v that configures max nodes in cluster to value %v", instascaleConfigmap.Name, instascaleConfigmap.Namespace, maxScaleNodesAllowed) - } + maxScaleNodesAllowed = int(r.Config.MaxScaleoutAllowed) useMachineSets = true - ocmSecretExists := ocmSecretExists(r.OcmSecretNamespace) - if ocmSecretExists { - machinePoolsExists := machinePoolExists() - - if machinePoolsExists { - useMachineSets = false - klog.Infof("Using machine pools %v", machinePoolsExists) + if ocmSecretRef := r.Config.OCMSecretRef; ocmSecretRef != nil { + if ocmSecret, err := getOCMSecret(ocmSecretRef); err != nil { + return fmt.Errorf("error reading OCM Secret from ref %q: %w", ocmSecretRef, err) + } else if token := ocmSecret.Data["token"]; len(token) > 0 { + ocmToken = string(token) } else { - klog.Infof("Setting useMachineSets to %v", useMachineSets) + return fmt.Errorf("token is missing from OCM Secret %q", ocmSecretRef) + } + if ok, err := machinePoolExists(); err != nil { + return err + } else if ok { + useMachineSets = false + klog.Info("Using machine pools for cluster auto-scaling") } } @@ -183,17 +175,8 @@ func (r *AppWrapperReconciler) SetupWithManager(mgr ctrl.Manager) error { Complete(r) } -func ocmSecretExists(namespace string) bool { - instascaleOCMSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(context.Background(), "instascale-ocm-secret", metav1.GetOptions{}) - if err != nil { - klog.Errorf("Error getting instascale-ocm-secret from namespace %v: %v", namespace, err) - klog.Infof("If you are looking to use OCM, ensure that the 'instascale-ocm-secret' secret is available on the cluster within namespace %v", namespace) - klog.Infof("Setting useMachineSets to %v.", useMachineSets) - return false - } - - ocmToken = string(instascaleOCMSecret.Data["token"]) - return true +func getOCMSecret(secretRef *corev1.SecretReference) (*corev1.Secret, error) { + return kubeClient.CoreV1().Secrets(secretRef.Namespace).Get(context.Background(), secretRef.Name, metav1.GetOptions{}) } func addAppwrappersThatNeedScaling() { diff --git a/controllers/machinepools.go b/controllers/machinepools.go index edf07b2..8bbe7e1 100644 --- a/controllers/machinepools.go +++ b/controllers/machinepools.go @@ -3,14 +3,16 @@ package controllers import ( "context" "fmt" + "os" + "strings" + ocmsdk "github.com/openshift-online/ocm-sdk-go" cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1" configv1 "github.com/openshift/api/config/v1" arbv1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + "k8s.io/apimachinery/pkg/types" "k8s.io/klog" - "os" - "strings" ) func createOCMConnection() (*ocmsdk.Connection, error) { @@ -82,17 +84,15 @@ func deleteMachinePool(aw *arbv1.AppWrapper) { }) } -// Check if machine pools exist -func machinePoolExists() bool { +func machinePoolExists() (bool, error) { connection, err := createOCMConnection() if err != nil { - fmt.Fprintf(os.Stderr, "Error creating OCM connection: %v", err) + return false, fmt.Errorf("error creating OCM connection: %w", err) } defer connection.Close() machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ocmClusterID).MachinePools() - klog.Infof("Machine pools: %v", machinePools) - return machinePools != nil + return machinePools != nil, nil } // getOCMClusterID determines the internal clusterID to be used for OCM API calls diff --git a/main.go b/main.go index f86026e..a333353 100644 --- a/main.go +++ b/main.go @@ -19,21 +19,28 @@ package main import ( "flag" "os" - - // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) - // to ensure that exec-entrypoint and run can make use of them. - _ "k8s.io/client-go/plugin/pkg/client/auth" + "strconv" configv1 "github.com/openshift/api/config/v1" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/client-go/kubernetes" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) + // to ensure that exec-entrypoint and run can make use of them. + _ "k8s.io/client-go/plugin/pkg/client/auth" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" "github.com/project-codeflare/instascale/controllers" - //+kubebuilder:scaffold:imports + "github.com/project-codeflare/instascale/pkg/config" + // +kubebuilder:scaffold:imports mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" ) @@ -46,7 +53,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(scheme)) utilruntime.Must(configv1.Install(scheme)) - //+kubebuilder:scaffold:scheme + // +kubebuilder:scaffold:scheme _ = mcadv1beta1.AddToScheme(scheme) } @@ -71,7 +78,47 @@ func main() { ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts))) - mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{ + ctx := ctrl.SetupSignalHandler() + + restConfig := ctrl.GetConfigOrDie() + kubeClient, err := kubernetes.NewForConfig(restConfig) + if err != nil { + setupLog.Error(err, "Error creating Kubernetes client") + os.Exit(1) + } + + // InstaScale configuration + cfg := config.InstaScaleConfiguration{ + MaxScaleoutAllowed: 3, + } + + // Source InstaScale ConfigMap + if InstaScaleConfigMap, err := kubeClient.CoreV1().ConfigMaps(configsNamespace).Get(ctx, "instascale-config", metav1.GetOptions{}); err != nil { + setupLog.Error(err, "Error reading 'instascale-config' ConfigMap") + os.Exit(1) + } else if maxScaleoutAllowed := InstaScaleConfigMap.Data["maxScaleoutAllowed"]; maxScaleoutAllowed != "" { + if max, err := strconv.Atoi(maxScaleoutAllowed); err != nil { + setupLog.Error(err, "Error converting 'maxScaleoutAllowed' to integer") + os.Exit(1) + } else { + cfg.MaxScaleoutAllowed = int32(max) + } + } + + // Source OCM Secret optionally + if OCMSecret, err := kubeClient.CoreV1().Secrets(ocmSecretNamespace).Get(ctx, "instascale-ocm-secret", metav1.GetOptions{}); apierrors.IsNotFound(err) { + setupLog.Info("If you are looking to use OCM, ensure the 'instascale-ocm-secret' Secret has been created") + } else if err != nil { + setupLog.Error(err, "Error checking if OCM Secret exists") + os.Exit(1) + } else { + cfg.OCMSecretRef = &corev1.SecretReference{ + Namespace: OCMSecret.Namespace, + Name: OCMSecret.Name, + } + } + + mgr, err := ctrl.NewManager(restConfig, ctrl.Options{ Scheme: scheme, MetricsBindAddress: metricsAddr, Port: 9443, @@ -85,15 +132,14 @@ func main() { } if err = (&controllers.AppWrapperReconciler{ - Client: mgr.GetClient(), - Scheme: mgr.GetScheme(), - ConfigsNamespace: configsNamespace, - OcmSecretNamespace: ocmSecretNamespace, + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + Config: cfg, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "AppWrapper") os.Exit(1) } - //+kubebuilder:scaffold:builder + // +kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { setupLog.Error(err, "unable to set up health check") @@ -105,7 +151,7 @@ func main() { } setupLog.Info("starting manager") - if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { + if err := mgr.Start(ctx); err != nil { setupLog.Error(err, "problem running manager") os.Exit(1) } diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 0000000..0fa6c0b --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,30 @@ +/* +Copyright 2022. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import corev1 "k8s.io/api/core/v1" + +// InstaScaleConfiguration defines the InstaScale configuration. +type InstaScaleConfiguration struct { + // ocmSecretRef is reference to the authentication Secret for connecting to OCM. + // If provided, MachinePools are used to auto-scale the cluster. + // +optional + OCMSecretRef *corev1.SecretReference `json:"ocmSecretRef,omitempty"` + // maxScaleoutAllowed defines the upper limit for the number of cluster nodes + // that can be scaled out by InstaScale. + MaxScaleoutAllowed int32 `json:"maxScaleoutAllowed"` +}