Refactor metrics merging to work with endpoints-controller

Previously, all metrics configuration was dealt with in the mutating
webhook handler. Now, since service/proxy registration happens in
endpoints-controller, some of the metrics configuration needs to be used
in endpoints-controller as well. There is still some functionality in
the handler that also needs metrics configuration, such as adding
prometheus annotations and running the consul sidecar.

This refactor pulls out common configuration for metrics into an
InjectConfiguration struct and adds the methods for getting values from
flags and annotations to that struct, so it can be commonly used between
endpoints-controller and the webhook handler.

connect-inject/common_inject_configuration.go

@@ -0,0 +1,160 @@
+package connectinject
+
+import (
+	"fmt"
+	"strconv"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+type InjectConfiguration struct {
+	DefaultEnableMetrics        bool
+	DefaultEnableMetricsMerging bool
+	DefaultMergedMetricsPort    string
+	DefaultPrometheusScrapePort string
+	DefaultPrometheusScrapePath string
+}
+
+// enableMetrics returns the default value in the handler, or overrides that
+// with the annotation if provided.
+func (ic InjectConfiguration) enableMetrics(pod corev1.Pod) (bool, error) {
+	enabled := ic.DefaultEnableMetrics
+	if raw, ok := pod.Annotations[annotationEnableMetrics]; ok && raw != "" {
+		enableMetrics, err := strconv.ParseBool(raw)
+		if err != nil {
+			return false, fmt.Errorf("%s annotation value of %s was invalid: %s", annotationEnableMetrics, raw, err)
+		}
+		enabled = enableMetrics
+	}
+	return enabled, nil
+}
+
+// enableMetricsMerging returns the default value in the handler, or overrides
+// that with the annotation if provided.
+func (ic InjectConfiguration) enableMetricsMerging(pod corev1.Pod) (bool, error) {
+	enabled := ic.DefaultEnableMetricsMerging
+	if raw, ok := pod.Annotations[annotationEnableMetricsMerging]; ok && raw != "" {
+		enableMetricsMerging, err := strconv.ParseBool(raw)
+		if err != nil {
+			return false, fmt.Errorf("%s annotation value of %s was invalid: %s", annotationEnableMetricsMerging, raw, err)
+		}
+		enabled = enableMetricsMerging
+	}
+	return enabled, nil
+}
+
+// mergedMetricsPort returns the default value in the handler, or overrides
+// that with the annotation if provided.
+func (ic InjectConfiguration) mergedMetricsPort(pod corev1.Pod) (string, error) {
+	return determineAndValidatePort(pod, annotationMergedMetricsPort, ic.DefaultMergedMetricsPort, false)
+}
+
+// prometheusScrapePort returns the default value in the handler, or overrides
+// that with the annotation if provided.
+func (ic InjectConfiguration) prometheusScrapePort(pod corev1.Pod) (string, error) {
+	return determineAndValidatePort(pod, annotationPrometheusScrapePort, ic.DefaultPrometheusScrapePort, false)
+}
+
+// prometheusScrapePath returns the default value in the handler, or overrides
+// that with the annotation if provided.
+func (ic InjectConfiguration) prometheusScrapePath(pod corev1.Pod) string {
+	if raw, ok := pod.Annotations[annotationPrometheusScrapePath]; ok && raw != "" {
+		return raw
+	}
+
+	return ic.DefaultPrometheusScrapePath
+}
+
+// serviceMetricsPort returns the port the service exposes metrics on. This will
+// default to the port used to register the service with Consul, and can be
+// overridden with the annotation if provided.
+func (ic InjectConfiguration) serviceMetricsPort(pod corev1.Pod) (string, error) {
+	// The annotationPort is the port used to register the service with Consul.
+	// If that has been set, it'll be used as the port for getting service
+	// metrics as well, unless overridden by the service-metrics-port annotation.
+	if raw, ok := pod.Annotations[annotationPort]; ok && raw != "" {
+		// The service metrics port can be privileged if the service author has
+		// written their service in such a way that it expects to be able to use
+		// privileged ports. So, the port metrics are exposed on the service can
+		// be privileged.
+		return determineAndValidatePort(pod, annotationServiceMetricsPort, raw, true)
+	}
+
+	// If the annotationPort is not set, the serviceMetrics port will be 0
+	// unless overridden by the service-metrics-port annotation. If the service
+	// metrics port is 0, the consul sidecar will not run a merged metrics
+	// server.
+	return determineAndValidatePort(pod, annotationServiceMetricsPort, "0", true)
+}
+
+// serviceMetricsPath returns a default of /metrics, or overrides
+// that with the annotation if provided.
+func (ic InjectConfiguration) serviceMetricsPath(pod corev1.Pod) string {
+	if raw, ok := pod.Annotations[annotationServiceMetricsPath]; ok && raw != "" {
+		return raw
+	}
+
+	return defaultServiceMetricsPath
+}
+
+// shouldRunMergedMetricsServer returns whether we need to run a merged metrics
+// server. This is used to configure the consul sidecar command, and the init
+// container, so it can pass appropriate arguments to the consul connect envoy
+// command.
+func (ic InjectConfiguration) shouldRunMergedMetricsServer(pod corev1.Pod) (bool, error) {
+	enableMetrics, err := ic.enableMetrics(pod)
+	if err != nil {
+		return false, err
+	}
+	enableMetricsMerging, err := ic.enableMetricsMerging(pod)
+	if err != nil {
+		return false, err
+	}
+	serviceMetricsPort, err := ic.serviceMetricsPort(pod)
+	if err != nil {
+		return false, err
+	}
+
+	// Don't need to check error here since serviceMetricsPort has been
+	// validated by calling ic.serviceMetricsPort above
+	smp, _ := strconv.Atoi(serviceMetricsPort)
+
+	if enableMetrics && enableMetricsMerging && smp > 0 {
+		return true, nil
+	}
+	return false, nil
+}
+
+// determineAndValidatePort behaves as follows:
+// If the annotation exists, validate the port and return it.
+// If the annotation does not exist, return the default port.
+// If the privileged flag is true, it will allow the port to be in the
+// privileged port range of 1-1023. Otherwise, it will only allow ports in the
+// unprivileged range of 1024-65535.
+func determineAndValidatePort(pod corev1.Pod, annotation string, defaultPort string, privileged bool) (string, error) {
+	if raw, ok := pod.Annotations[annotation]; ok && raw != "" {
+		port, err := portValue(pod, raw)
+		if err != nil {
+			return "", fmt.Errorf("%s annotation value of %s is not a valid integer", annotation, raw)
+		}
+
+		if privileged && (port < 1 || port > 65535) {
+			return "", fmt.Errorf("%s annotation value of %d is not in the valid port range 1-65535", annotation, port)
+		} else if !privileged && (port < 1024 || port > 65535) {
+			return "", fmt.Errorf("%s annotation value of %d is not in the unprivileged port range 1024-65535", annotation, port)
+		}
+
+		// if the annotation exists, return the validated port
+		return fmt.Sprint(port), nil
+	}
+
+	// if the annotation does not exist, return the default
+	if defaultPort != "" {
+		port, err := portValue(pod, defaultPort)
+		if err != nil {
+			return "", fmt.Errorf("%s is not a valid port on the pod %s", defaultPort, pod.Name)
+		}
+		return fmt.Sprint(port), nil
+	}
+	return "", nil
+}