Adding controller for ssp-operator-metrics service

Adding a controller to add and reconcile a Service that should exist
independently from the SSP CR for Prometheus monitoring to work with SSP.

Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2076790

Co-authored-by: Karel Simon <ksimon@redhat.com>
Signed-off-by: borod108 <boris.od@gmail.com>

config/default/kustomization.yaml

@@ -23,7 +23,7 @@ bases:
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
 # [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
-- ../prometheus
+#- ../prometheus
 
 patchesStrategicMerge:
 # Protect the /metrics endpoint by putting it behind auth.

config/rbac/role.yaml

@@ -15,6 +15,17 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - update
+  - watch
 - apiGroups:
   - admissionregistration.k8s.io
   resources:

controllers/services_controller.go

@@ -0,0 +1,181 @@
+package controllers
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"github.com/go-logr/logr"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"kubevirt.io/ssp-operator/internal/common"
+	"kubevirt.io/ssp-operator/internal/operands/metrics"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	MetricsServiceName = "ssp-operator-metrics"
+	OperatorName       = "ssp-operator"
+)
+
+func ServiceObject(namespace string) *v1.Service {
+	policyCluster := v1.ServiceInternalTrafficPolicyCluster
+	familyPolicy := v1.IPFamilyPolicySingleStack
+	return &v1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      MetricsServiceName,
+			Namespace: namespace,
+			Labels: map[string]string{
+				metrics.PrometheusLabelKey: metrics.PrometheusLabelValue,
+			},
+		},
+		Spec: v1.ServiceSpec{
+			InternalTrafficPolicy: &policyCluster,
+			IPFamilies:            []v1.IPFamily{v1.IPv4Protocol},
+			IPFamilyPolicy:        &familyPolicy,
+			Ports: []v1.ServicePort{
+				{
+					Name:       metrics.MetricsPortName,
+					Port:       443,
+					Protocol:   v1.ProtocolTCP,
+					TargetPort: intstr.FromString(metrics.MetricsPortName),
+				},
+			},
+			Selector: map[string]string{
+				metrics.PrometheusLabelKey: metrics.PrometheusLabelValue,
+				"name":                     OperatorName,
+			},
+			SessionAffinity: v1.ServiceAffinityNone,
+			Type:            v1.ServiceTypeClusterIP,
+		},
+	}
+}
+
+// Annotation to generate RBAC roles to read and modify services
+// +kubebuilder:rbac:groups="",resources=services,verbs=get;watch;list;create;update;delete
+
+func CreateServiceController(mgr ctrl.Manager) (*serviceReconciler, error) {
+	return newServiceReconciler(mgr)
+}
+
+func (r *serviceReconciler) Start(ctx context.Context, mgr ctrl.Manager) error {
+	err := r.createMetricsService(ctx)
+	if err != nil && !errors.IsAlreadyExists(err) {
+		return fmt.Errorf("error start serviceReconciler: %w", err)
+	}
+
+	return r.setupController(mgr)
+}
+
+func (r *serviceReconciler) setServiceOwnerReference(service *v1.Service) error {
+	return controllerutil.SetOwnerReference(r.deployment, service, r.client.Scheme())
+}
+
+func (r *serviceReconciler) createMetricsService(ctx context.Context) error {
+	service := ServiceObject(r.serviceNamespace)
+	err := r.setServiceOwnerReference(service)
+	if err != nil {
+		return fmt.Errorf("error setting owner reference: %w", err)
+	}
+	return r.client.Create(ctx, service)
+}
+
+func (r *serviceReconciler) setupController(mgr ctrl.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		Named("service-controller").
+		For(&v1.Service{}, builder.WithPredicates(predicate.NewPredicateFuncs(
+			func(object client.Object) bool {
+				return object.GetName() == MetricsServiceName && object.GetNamespace() == r.serviceNamespace
+			}))).
+		Complete(r)
+}
+
+// serviceReconciler reconciles the required services in the operator's namespace
+type serviceReconciler struct {
+	client           client.Client
+	log              logr.Logger
+	serviceNamespace string
+	deployment       *apps.Deployment
+}
+
+func getOperatorDeployment(namespace string, apiReader client.Reader) (*apps.Deployment, error) {
+	objKey := client.ObjectKey{Namespace: namespace, Name: OperatorName}
+	var deployment apps.Deployment
+	err := apiReader.Get(context.TODO(), objKey, &deployment)
+	if err != nil {
+		return nil, fmt.Errorf("getOperatorDeployment, get deployment: %w", err)
+	}
+	return &deployment, nil
+}
+
+func newServiceReconciler(mgr ctrl.Manager) (*serviceReconciler, error) {
+	logger := ctrl.Log.WithName("controllers").WithName("Resources")
+	namespace, err := getOperatorNamespace(logger)
+	if err != nil {
+		return nil, fmt.Errorf("in newServiceReconciler: %w", err)
+	}
+
+	deployment, err := getOperatorDeployment(namespace, mgr.GetAPIReader())
+	if err != nil {
+		return nil, fmt.Errorf("in newServiceReconciler: %w", err)
+	}
+
+	reconciler := &serviceReconciler{
+		client:           mgr.GetClient(),
+		log:              logger,
+		serviceNamespace: namespace,
+		deployment:       deployment,
+	}
+
+	return reconciler, nil
+}
+
+func getOperatorNamespace(logger logr.Logger) (string, error) {
+	nsBytes, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/namespace")
+	if err != nil {
+		return "", fmt.Errorf("in getOperatorNamespace failed in call to downward API: %w", err)
+	}
+	ns := strings.TrimSpace(string(nsBytes))
+	logger.Info("Found namespace", "Namespace", ns)
+	return ns, nil
+}
+
+func (r *serviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
+	r.log.Info("Starting service reconciliation...", "request", req.String())
+	service := ServiceObject(req.Namespace)
+	var foundService v1.Service
+	foundService.Name = service.Name
+	foundService.Namespace = service.Namespace
+
+	_, err = controllerutil.CreateOrUpdate(ctx, r.client, &foundService, func() error {
+		if !foundService.GetDeletionTimestamp().IsZero() {
+			// Skip update, because the resource is being deleted
+			return nil
+		}
+
+		clusterIP := foundService.Spec.ClusterIP
+		foundService.Spec = service.Spec
+		foundService.Spec.ClusterIP = clusterIP
+
+		common.UpdateLabels(service, &foundService)
+
+		err = r.setServiceOwnerReference(&foundService)
+		if err != nil {
+			return fmt.Errorf("error at setServiceOwnerReference: %w", err)
+		}
+		return nil
+	})
+
+	return ctrl.Result{}, err
+}
+
+var _ reconcile.Reconciler = &serviceReconciler{}

controllers/setup.go

@@ -2,6 +2,7 @@ package controllers
 
 import (
 	"context"
+	"fmt"
 	"path/filepath"
 
 	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
@@ -50,6 +51,25 @@ func CreateAndSetupReconciler(mgr controllerruntime.Manager) error {
 		return err
 	}
 
+	serviceController, err := CreateServiceController(mgr)
+	if err != nil {
+		return err
+	}
+
+	err = mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
+		err := serviceController.Start(ctx, mgr)
+		if err != nil {
+			return fmt.Errorf("error adding serviceController: %w", err)
+		}
+
+		mgr.GetLogger().Info("Services Controller started")
+
+		return nil
+	}))
+	if err != nil {
+		return err
+	}
+
 	reconciler := NewSspReconciler(mgr.GetClient(), mgr.GetAPIReader(), infrastructureTopology, sspOperands)
 
 	if requiredCrdsExist(requiredCrds, crdList.Items) {

data/olm-catalog/ssp-operator.clusterserviceversion.yaml

@@ -59,6 +59,17 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - ""
+          resources:
+          - services
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - update
+          - watch
         - apiGroups:
           - admissionregistration.k8s.io
           resources:

internal/common/resource.go

@@ -175,7 +175,7 @@ func (r *reconcileBuilder) Reconcile() (ReconcileResult, error) {
 		// if that is not correct, this code needs to be changed.
 		found.SetOwnerReferences(r.resource.GetOwnerReferences())
 
-		updateLabels(r.resource, found)
+		UpdateLabels(r.resource, found)
 		updateAnnotations(r.resource, found)
 		if r.options.AlwaysCallUpdateFunc || !r.request.VersionCache.Contains(found) {
 			// The generation was updated by other cluster components,
@@ -356,7 +356,7 @@ func updateAnnotations(expected, found client.Object) {
 	updateStringMap(expected.GetAnnotations(), found.GetAnnotations())
 }
 
-func updateLabels(expected, found client.Object) {
+func UpdateLabels(expected, found client.Object) {
 	if found.GetLabels() == nil {
 		found.SetLabels(expected.GetLabels())
 		return

tests/service_controller_test.go

@@ -0,0 +1,77 @@
+package tests
+
+import (
+	"fmt"
+	"reflect"
+	"time"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"kubevirt.io/ssp-operator/controllers"
+	"kubevirt.io/ssp-operator/internal/operands/metrics"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func getSspMetricsService() (*v1.Service, error) {
+	service := controllers.ServiceObject(strategy.GetSSPDeploymentNameSpace())
+	err := apiClient.Get(ctx, client.ObjectKeyFromObject(service), service)
+	return service, err
+}
+
+func equalService(serviceA, serviceB *v1.Service) bool {
+	return reflect.DeepEqual(serviceA.Labels, serviceB.Labels) && reflect.DeepEqual(serviceA.Spec, serviceB.Spec)
+}
+
+var _ = Describe("Service Controller", func() {
+	BeforeEach(func() {
+		waitUntilDeployed()
+	})
+
+	It("[test_id: 8807] Should create ssp-operator-metrics service", func() {
+		_, serviceErr := getSspMetricsService()
+		Expect(serviceErr).ToNot(HaveOccurred(), "Failed to get ssp-operator-metrics service")
+	})
+
+	It("[test_id: 8808] Should re-create ssp-operator-metrics service if deleted", func() {
+		service, serviceErr := getSspMetricsService()
+		Expect(serviceErr).ToNot(HaveOccurred(), "Failed to get ssp-operator-metrics service")
+		oldUID := service.UID
+		Expect(apiClient.Delete(ctx, service)).To(Succeed())
+		Eventually(func() (types.UID, error) {
+			var foundService v1.Service
+			err := apiClient.Get(ctx, client.ObjectKeyFromObject(service), &foundService)
+			if err != nil {
+				return "", err
+			}
+			return foundService.UID, nil
+		}, shortTimeout, time.Second).ShouldNot(Equal(oldUID), fmt.Sprintf("Did not recreate the %s service", controllers.MetricsServiceName))
+	})
+
+	It("[test_id: 8810] Should restore ssp-operator-metrics service after update", func() {
+		service, serviceErr := getSspMetricsService()
+		Expect(serviceErr).ToNot(HaveOccurred(), "Failed to get ssp-operator-metrics service")
+		changed := service.DeepCopy()
+		changed.Labels = nil
+		changed.Spec.Ports = []v1.ServicePort{
+			{
+				Name:       metrics.MetricsPortName,
+				Port:       755,
+				Protocol:   v1.ProtocolTCP,
+				TargetPort: intstr.FromString(metrics.MetricsPortName),
+			},
+		}
+
+		Eventually(func() error {
+			return apiClient.Update(ctx, changed)
+		}, shortTimeout, time.Second).Should(Succeed())
+
+		Eventually(func() bool {
+			Expect(apiClient.Get(ctx, client.ObjectKeyFromObject(changed), changed)).ToNot(HaveOccurred())
+			return equalService(service, changed)
+		}, shortTimeout, time.Second).Should(BeTrue())
+	})
+})