Controller-Manager: propagate virtual kubelet metrics settings

cmd/liqo-controller-manager/main.go

@@ -12,6 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+// Package main contains the main function for the Liqo controller manager.
 package main
 
 import (
@@ -72,6 +73,7 @@ import (
 	liqoerrors "github.com/liqotech/liqo/pkg/utils/errors"
 	"github.com/liqotech/liqo/pkg/utils/mapper"
 	"github.com/liqotech/liqo/pkg/utils/restcfg"
+	"github.com/liqotech/liqo/pkg/vkMachinery"
 	"github.com/liqotech/liqo/pkg/vkMachinery/forge"
 )
 
@@ -96,6 +98,8 @@ func main() {
 	var nodeExtraAnnotations, nodeExtraLabels argsutils.StringMap
 	var kubeletCPURequests, kubeletCPULimits argsutils.Quantity
 	var kubeletRAMRequests, kubeletRAMLimits argsutils.Quantity
+	var kubeletMetricsAddress string
+	var kubeletMetricsEnabled bool
 
 	webhookPort := flag.Uint("webhook-port", 9443, "The port the webhook server binds to")
 	metricsAddr := flag.String("metrics-address", ":8080", "The address the metric endpoint binds to")
@@ -152,6 +156,8 @@ func main() {
 	flag.Var(&kubeletCPULimits, "kubelet-cpu-limits", "CPU limits assigned to the Virtual Kubelet Pod")
 	flag.Var(&kubeletRAMRequests, "kubelet-ram-requests", "RAM requests assigned to the Virtual Kubelet Pod")
 	flag.Var(&kubeletRAMLimits, "kubelet-ram-limits", "RAM limits assigned to the Virtual Kubelet Pod")
+	flag.StringVar(&kubeletMetricsAddress, "kubelet-metrics-address", vkMachinery.MetricsAddress, "The address the kubelet metrics endpoint binds to")
+	flag.BoolVar(&kubeletMetricsEnabled, "kubelet-metrics-enabled", false, "Enable the kubelet metrics endpoint")
 	flag.Var(&nodeExtraAnnotations, "node-extra-annotations", "Extra annotations to add to the Virtual Node")
 	flag.Var(&nodeExtraLabels, "node-extra-labels", "Extra labels to add to the Virtual Node")
 	kubeletIpamServer := flag.String("kubelet-ipam-server", "",
@@ -310,6 +316,8 @@ func main() {
 		LimitsCPU:            kubeletCPULimits.Quantity,
 		LimitsRAM:            kubeletRAMLimits.Quantity,
 		IpamEndpoint:         *kubeletIpamServer,
+		MetricsAddress:       kubeletMetricsAddress,
+		MetricsEnabled:       kubeletMetricsEnabled,
 	}
 
 	resourceOfferReconciler := resourceoffercontroller.NewResourceOfferController(

cmd/virtual-kubelet/root/flag.go

@@ -72,7 +72,8 @@ func InstallFlags(flags *pflag.FlagSet, o *Opts) {
 	flags.BoolVar(&o.EnableStorage, "enable-storage", false, "Enable the Liqo storage reflection")
 	flags.StringVar(&o.VirtualStorageClassName, "virtual-storage-class-name", "liqo", "Name of the virtual storage class")
 	flags.StringVar(&o.RemoteRealStorageClassName, "remote-real-storage-class-name", "", "Name of the real storage class to use for the actual volumes")
-	flags.BoolVar(&o.EnableMetrics, "enable-metrics", false, "Enable the metrics server")
+	flags.BoolVar(&o.EnableMetrics, "metrics-enabled", false, "Enable the metrics server")
+	flags.StringVar(&o.MetricsAddress, "metrics-address", ":8080", "The address to listen to for metrics requests")
 	flags.StringVar(&o.HomeAPIServerHost, "home-api-server-host", "",
 		"Home cluster API server HOST, this parameter is optional and required only to override the default values")
 	flags.StringVar(&o.HomeAPIServerPort, "home-api-server-port", "",

cmd/virtual-kubelet/root/opts.go

@@ -97,6 +97,7 @@ type Opts struct {
 	VirtualStorageClassName    string
 	RemoteRealStorageClassName string
 	EnableMetrics              bool
+	MetricsAddress             string
 
 	HomeAPIServerHost string
 	HomeAPIServerPort string

cmd/virtual-kubelet/root/root.go

@@ -197,7 +197,7 @@ func runRootCommand(ctx context.Context, c *Opts) error {
 	}
 
 	if c.EnableMetrics {
-		metrics.SetupMetricHandler()
+		metrics.SetupMetricHandler(c.MetricsAddress)
 	}
 
 	go func() {

deployments/liqo/README.md

@@ -138,7 +138,7 @@
 | virtualKubelet.metrics.podMonitor.enabled | bool | `false` |  |
 | virtualKubelet.metrics.podMonitor.interval | string | `""` |  |
 | virtualKubelet.metrics.podMonitor.scrapeTimeout | string | `""` |  |
-| virtualKubelet.metrics.port | int | `9090` | port used to expose metrics. |
+| virtualKubelet.metrics.port | int | `5872` | port used to expose metrics. |
 | virtualKubelet.virtualNode.extra.annotations | object | `{}` | virtual node extra annotations |
 | virtualKubelet.virtualNode.extra.labels | object | `{}` | virtual node extra labels |
 | webhook.failurePolicy | string | `"Fail"` | the webhook failure policy, among Ignore and Fail |

deployments/liqo/templates/liqo-controller-manager-deployment.yaml

@@ -26,9 +26,6 @@
 {{- $vkargs = append $vkargs "--certificate-type=aws" }}
 {{- end }}
 {{- end }}
-{{- if not (or (has "--enable-metrics" $vkargs ) (has "--enable-metrics=true" $vkargs ) (has "--enable-metrics=false" $vkargs )) }}
-{{- $vkargs = append $vkargs "--enable-metrics=true" }}
-{{- end}}
 
 apiVersion: apps/v1
 kind: Deployment
@@ -81,6 +78,11 @@ spec:
           - --enable-incoming-peering={{ .Values.discovery.config.incomingPeeringEnabled }}
           - --resource-sharing-percentage={{ .Values.controllerManager.config.resourceSharingPercentage }}
           - --kubelet-image={{ .Values.virtualKubelet.imageName }}{{ include "liqo.suffix" $ctrlManagerConfig }}:{{ include "liqo.version" $ctrlManagerConfig }}
+          {{- if .Values.virtualKubelet.metrics.enabled }}
+          - --kubelet-metrics-address=:{{ .Values.virtualKubelet.metrics.port }}
+          - --kubelet-metrics-enabled={{ .Values.virtualKubelet.metrics.enabled }}
+          {{- else }}
+          {{- end }}
           {{- if .Values.networkManager.externalIPAM.enabled }}
           - --kubelet-ipam-server={{ .Values.networkManager.externalIPAM.url }}
           {{- else if not .Values.networking.internal }}

deployments/liqo/values.yaml

@@ -319,7 +319,7 @@ virtualKubelet:
     # -- expose metrics about virtual kubelet resources.
     enabled: false
     # -- port used to expose metrics.
-    port: 9090
+    port: 5872
     podMonitor:
       # # -- create a prometheus podmonitor.
       enabled: false

pkg/liqo-controller-manager/resourceoffer-controller/resourceoffer_controller_methods.go

@@ -118,7 +118,7 @@ func (r *ResourceOfferReconciler) createVirtualKubeletDeployment(
 
 	// forge the virtual Kubelet
 	vkDeployment, err := forge.VirtualKubeletDeployment(
-		&r.cluster, &remoteClusterIdentity, namespace, r.liqoNamespace,
+		&r.cluster, &remoteClusterIdentity, namespace,
 		r.virtualKubeletOpts, resourceOffer)
 	if err != nil {
 		klog.Error(err)

pkg/virtualKubelet/metrics/metrics.go

@@ -24,11 +24,6 @@ import (
 	"k8s.io/klog/v2"
 )
 
-const (
-	// MetricsPort is the metrics port constant.
-	MetricsPort = ":9090"
-)
-
 var (
 	// ErrorsCounter is the counter of the errors occurred during the reflection.
 	ErrorsCounter *prometheus.CounterVec
@@ -59,7 +54,7 @@ func init() {
 }
 
 // SetupMetricHandler sets up the metric handler.
-func SetupMetricHandler() {
+func SetupMetricHandler(metricsAddress string) {
 	// Register the metrics to the prometheus registry.
 	prometheus.MustRegister(ErrorsCounter)
 	// Register the metrics to the prometheus registry.
@@ -68,10 +63,10 @@ func SetupMetricHandler() {
 	http.Handle("/metrics", promhttp.Handler())
 
 	go func() {
-		klog.Infof("Starting the virtual kubelet Metric Handler listening on %q", MetricsPort)
+		klog.Infof("Starting the virtual kubelet Metric Handler listening on %q", metricsAddress)
 
 		server := &http.Server{
-			Addr:              ":1234",
+			Addr:              metricsAddress,
 			ReadHeaderTimeout: 10 * time.Second,
 		}
 

pkg/vkMachinery/const.go

@@ -39,3 +39,6 @@ var KubeletBaseLabels = map[string]string{
 var ClusterRoleBindingLabels = map[string]string{
 	"app.kubernetes.io/managed-by": "advertisementoperator",
 }
+
+// MetricsAddress is the default address used to expose metrics.
+const MetricsAddress = ":8080"

pkg/vkMachinery/forge/creation.go

@@ -29,7 +29,7 @@ import (
 )
 
 // VirtualKubeletDeployment forges the deployment for a virtual-kubelet.
-func VirtualKubeletDeployment(homeCluster, remoteCluster *discoveryv1alpha1.ClusterIdentity, vkNamespace, liqoNamespace string,
+func VirtualKubeletDeployment(homeCluster, remoteCluster *discoveryv1alpha1.ClusterIdentity, vkNamespace string,
 	opts *VirtualKubeletOpts, resourceOffer *sharingv1alpha1.ResourceOffer) (*appsv1.Deployment, error) {
 	vkLabels := VirtualKubeletLabels(remoteCluster.ClusterID, opts)
 	annotations := opts.ExtraAnnotations
@@ -49,7 +49,7 @@ func VirtualKubeletDeployment(homeCluster, remoteCluster *discoveryv1alpha1.Clus
 					Labels:      vkLabels,
 					Annotations: annotations,
 				},
-				Spec: forgeVKPodSpec(vkNamespace, liqoNamespace, homeCluster, remoteCluster, opts, resourceOffer),
+				Spec: forgeVKPodSpec(vkNamespace, homeCluster, remoteCluster, opts, resourceOffer),
 			},
 		},
 	}, nil

pkg/vkMachinery/forge/forge.go

@@ -16,6 +16,8 @@ package forge
 
 import (
 	"fmt"
+	"strconv"
+	"strings"
 
 	v1 "k8s.io/api/core/v1"
 
@@ -37,7 +39,7 @@ func getDefaultStorageClass(storageClasses []sharingv1alpha1.StorageType) sharin
 
 func forgeVKContainers(
 	vkImage string, homeCluster, remoteCluster *discoveryv1alpha1.ClusterIdentity,
-	nodeName, vkNamespace, liqoNamespace string, opts *VirtualKubeletOpts,
+	nodeName, vkNamespace string, opts *VirtualKubeletOpts,
 	resourceOffer *sharingv1alpha1.ResourceOffer) []v1.Container {
 	command := []string{
 		"/usr/bin/virtual-kubelet",
@@ -72,6 +74,24 @@ func forgeVKContainers(
 
 	args = append(args, opts.ExtraArgs...)
 
+	containerPorts := []v1.ContainerPort{}
+	args = append(args, stringifyArgument("--metrics-enabled", strconv.FormatBool(opts.MetricsEnabled)))
+	if opts.MetricsEnabled {
+		args = append(args, stringifyArgument("--metrics-address", opts.MetricsAddress))
+		metrAddrSplit := strings.Split(opts.MetricsAddress, ":")
+		metricsPort, err := strconv.ParseInt(metrAddrSplit[len(metrAddrSplit)-1], 10, 32)
+		if err != nil {
+			metrAddrSplit := strings.Split(vk.MetricsAddress, ":")
+			// if the metrics address is not valid, use the default one
+			metricsPort, _ = strconv.ParseInt(metrAddrSplit[len(metrAddrSplit)-1], 10, 32)
+		}
+		containerPorts = append(containerPorts, v1.ContainerPort{
+			Name:          "metrics",
+			ContainerPort: int32(metricsPort),
+			Protocol:      v1.ProtocolTCP,
+		})
+	}
+
 	return []v1.Container{
 		{
 			Name:      "virtual-kubelet",
@@ -85,25 +105,19 @@ func forgeVKContainers(
 					ValueFrom: &v1.EnvVarSource{FieldRef: &v1.ObjectFieldSelector{FieldPath: "status.podIP"}},
 				},
 			},
-			Ports: []v1.ContainerPort{
-				{
-					Name:          "metrics",
-					ContainerPort: 9090,
-					Protocol:      v1.ProtocolTCP,
-				},
-			},
+			Ports: containerPorts,
 		},
 	}
 }
 
 func forgeVKPodSpec(
-	vkNamespace, liqoNamespace string,
+	vkNamespace string,
 	homeCluster, remoteCluster *discoveryv1alpha1.ClusterIdentity, opts *VirtualKubeletOpts,
 	resourceOffer *sharingv1alpha1.ResourceOffer) v1.PodSpec {
 	nodeName := virtualKubelet.VirtualNodeName(remoteCluster)
 	return v1.PodSpec{
 		Containers: forgeVKContainers(opts.ContainerImage, homeCluster, remoteCluster,
-			nodeName, vkNamespace, liqoNamespace, opts, resourceOffer),
+			nodeName, vkNamespace, opts, resourceOffer),
 		ServiceAccountName: vk.ServiceAccountName,
 	}
 }

pkg/vkMachinery/forge/type.go

@@ -34,4 +34,6 @@ type VirtualKubeletOpts struct {
 	RequestsRAM          resource.Quantity
 	LimitsRAM            resource.Quantity
 	IpamEndpoint         string
+	MetricsEnabled       bool
+	MetricsAddress       string
 }