Merge pull request #8511 from olemarkus/automated-cherry-pick-of-#843…

…3-origin-release-1.17 Automated cherry pick of #8433: Make it possible to enable Prometheus metrics for Cilium
kubernetes · Feb 7, 2020 · 2dc6b68 · 2dc6b68
2 parents e3e1b32 + 7eebe6b
commit 2dc6b68
Show file tree

Hide file tree

Showing 8 changed files with 74 additions and 6 deletions.
diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -2359,6 +2359,8 @@ spec:
                       items:
                         type: string
                       type: array
+                    agentPrometheusPort:
+                      type: integer
                     allowLocalhost:
                       type: string
                     autoDirectNodeRoutes:
@@ -2405,6 +2407,8 @@ spec:
                       type: boolean
                     enablePolicy:
                       type: string
+                    enablePrometheusMetrics:
+                      type: boolean
                     enableTracing:
                       type: boolean
                     enableipv4:

diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go
@@ -177,6 +177,7 @@ type CiliumNetworkingSpec struct {
 
 	AccessLog                string            `json:"accessLog,omitempty"`
 	AgentLabels              []string          `json:"agentLabels,omitempty"`
+	AgentPrometheusPort      int               `json:"agentPrometheusPort,omitempty"`
 	AllowLocalhost           string            `json:"allowLocalhost,omitempty"`
 	AutoIpv6NodeRoutes       bool              `json:"autoIpv6NodeRoutes,omitempty"`
 	BPFRoot                  string            `json:"bpfRoot,omitempty"`
@@ -190,6 +191,7 @@ type CiliumNetworkingSpec struct {
 	DisableK8sServices       bool              `json:"disableK8sServices,omitempty"`
 	EnablePolicy             string            `json:"enablePolicy,omitempty"`
 	EnableTracing            bool              `json:"enableTracing,omitempty"`
+	EnablePrometheusMetrics  bool              `json:"enablePrometheusMetrics,omitempty"`
 	EnvoyLog                 string            `json:"envoyLog,omitempty"`
 	Ipv4ClusterCIDRMaskSize  int               `json:"ipv4ClusterCidrMaskSize,omitempty"`
 	Ipv4Node                 string            `json:"ipv4Node,omitempty"`

diff --git a/pkg/apis/kops/v1alpha1/networking.go b/pkg/apis/kops/v1alpha1/networking.go
@@ -174,6 +174,7 @@ type CiliumNetworkingSpec struct {
 
 	AccessLog                string            `json:"accessLog,omitempty"`
 	AgentLabels              []string          `json:"agentLabels,omitempty"`
+	AgentPrometheusPort      int               `json:"agentPrometheusPort,omitempty"`
 	AllowLocalhost           string            `json:"allowLocalhost,omitempty"`
 	AutoIpv6NodeRoutes       bool              `json:"autoIpv6NodeRoutes,omitempty"`
 	BPFRoot                  string            `json:"bpfRoot,omitempty"`
@@ -186,6 +187,7 @@ type CiliumNetworkingSpec struct {
 	DisableIpv4              bool              `json:"disableIpv4,omitempty"`
 	DisableK8sServices       bool              `json:"disableK8sServices,omitempty"`
 	EnablePolicy             string            `json:"enablePolicy,omitempty"`
+	EnablePrometheusMetrics  bool              `json:"enablePrometheusMetrics,omitempty"`
 	EnableTracing            bool              `json:"enableTracing,omitempty"`
 	EnvoyLog                 string            `json:"envoyLog,omitempty"`
 	Ipv4ClusterCIDRMaskSize  int               `json:"ipv4ClusterCidrMaskSize,omitempty"`

diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go
@@ -175,6 +175,7 @@ type CiliumNetworkingSpec struct {
 
 	AccessLog                string            `json:"accessLog,omitempty"`
 	AgentLabels              []string          `json:"agentLabels,omitempty"`
+	AgentPrometheusPort      int               `json:"agentPrometheusPort,omitempty"`
 	AllowLocalhost           string            `json:"allowLocalhost,omitempty"`
 	AutoIpv6NodeRoutes       bool              `json:"autoIpv6NodeRoutes,omitempty"`
 	BPFRoot                  string            `json:"bpfRoot,omitempty"`
@@ -187,6 +188,7 @@ type CiliumNetworkingSpec struct {
 	DisableIpv4              bool              `json:"disableIpv4,omitempty"`
 	DisableK8sServices       bool              `json:"disableK8sServices,omitempty"`
 	EnablePolicy             string            `json:"enablePolicy,omitempty"`
+	EnablePrometheusMetrics  bool              `json:"enablePrometheusMetrics,omitempty"`
 	EnableTracing            bool              `json:"enableTracing,omitempty"`
 	EnvoyLog                 string            `json:"envoyLog,omitempty"`
 	Ipv4ClusterCIDRMaskSize  int               `json:"ipv4ClusterCidrMaskSize,omitempty"`

diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template
@@ -20,6 +20,15 @@ data:
   identity-allocation-mode: crd
   # If you want to run cilium in debug mode change this value to true
   debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}"
+  {{ if .EnablePrometheusMetrics }}
+  # If you want metrics enabled in all of your Cilium agents, set the port for
+  # which the Cilium agents will have their metrics exposed.
+  # This option deprecates the "prometheus-serve-addr" in the
+  # "cilium-metrics-config" ConfigMap
+  # NOTE that this will open the port on ALL nodes where Cilium pods are
+  # scheduled.
+  prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}"
+  {{ end }}
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
   enable-ipv4: "{{- if or  (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}"
@@ -385,6 +394,13 @@ spec:
           successThreshold: 1
           timeoutSeconds: 5
         name: cilium-agent
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: {{ or .AgentPrometheusPort "9090" }}
+          hostPort: {{ or .AgentPrometheusPort "9090" }}
+          name: prometheus
+          protocol: TCP
+        {{ end }}
         readinessProbe:
           exec:
             command:
@@ -543,6 +559,10 @@ spec:
       - args:
         - --debug=$(CILIUM_DEBUG)
         - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE)
+{{ with .Networking.Cilium }}
+        {{ if .EnablePrometheusMetrics }}
+        - --enable-metrics
+        {{ end }}
         command:
         - cilium-operator
         env:
@@ -622,12 +642,17 @@ spec:
               key: AWS_DEFAULT_REGION
               name: cilium-aws
               optional: true
-
-{{ with .Networking.Cilium }}
         image: "docker.io/cilium/operator:{{ .Version }}"
-{{ end }}
         imagePullPolicy: IfNotPresent
         name: cilium-operator
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: 6942
+          hostPort: 6942
+          name: prometheus
+          protocol: TCP
+        {{ end }}
+{{ end }}
         livenessProbe:
           httpGet:
             path: /healthz

diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template
@@ -20,6 +20,15 @@ data:
   identity-allocation-mode: crd
   # If you want to run cilium in debug mode change this value to true
   debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}"
+  {{ if .EnablePrometheusMetrics }}
+  # If you want metrics enabled in all of your Cilium agents, set the port for
+  # which the Cilium agents will have their metrics exposed.
+  # This option deprecates the "prometheus-serve-addr" in the
+  # "cilium-metrics-config" ConfigMap
+  # NOTE that this will open the port on ALL nodes where Cilium pods are
+  # scheduled.
+  prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}"
+  {{ end }}
   # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4
   # address.
   enable-ipv4: "{{- if or  (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}"
@@ -385,6 +394,13 @@ spec:
           successThreshold: 1
           timeoutSeconds: 5
         name: cilium-agent
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: {{ or .AgentPrometheusPort "9090" }}
+          hostPort: {{ or .AgentPrometheusPort "9090" }}
+          name: prometheus
+          protocol: TCP
+        {{ end }}
         readinessProbe:
           exec:
             command:
@@ -541,6 +557,10 @@ spec:
       containers:
       - args:
         - --debug=$(CILIUM_DEBUG)
+{{ with .Networking.Cilium }}
+        {{ if .EnablePrometheusMetrics }}
+        - --enable-metrics
+        {{ end }}
         command:
         - cilium-operator
         env:
@@ -614,12 +634,17 @@ spec:
               key: AWS_DEFAULT_REGION
               name: cilium-aws
               optional: true
-
-{{ with .Networking.Cilium }}
         image: "docker.io/cilium/operator:{{ .Version }}"
-{{ end }}
         imagePullPolicy: IfNotPresent
         name: cilium-operator
+        {{ if .EnablePrometheusMetrics }}
+        ports:
+        - containerPort: 6942
+          hostPort: 6942
+          name: prometheus
+          protocol: TCP
+        {{ end }}
+{{ end }}
         livenessProbe:
           httpGet:
             path: /healthz