diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml index 70ae83fbf9474..8801fc7fee0ea 100644 --- a/k8s/crds/kops.k8s.io_clusters.yaml +++ b/k8s/crds/kops.k8s.io_clusters.yaml @@ -2359,6 +2359,8 @@ spec: items: type: string type: array + agentPrometheusPort: + type: integer allowLocalhost: type: string autoDirectNodeRoutes: @@ -2405,6 +2407,8 @@ spec: type: boolean enablePolicy: type: string + enablePrometheusMetrics: + type: boolean enableTracing: type: boolean enableipv4: diff --git a/pkg/apis/kops/networking.go b/pkg/apis/kops/networking.go index f23f948fe7c35..67d94ed4070be 100644 --- a/pkg/apis/kops/networking.go +++ b/pkg/apis/kops/networking.go @@ -177,6 +177,7 @@ type CiliumNetworkingSpec struct { AccessLog string `json:"accessLog,omitempty"` AgentLabels []string `json:"agentLabels,omitempty"` + AgentPrometheusPort int `json:"agentPrometheusPort,omitempty"` AllowLocalhost string `json:"allowLocalhost,omitempty"` AutoIpv6NodeRoutes bool `json:"autoIpv6NodeRoutes,omitempty"` BPFRoot string `json:"bpfRoot,omitempty"` @@ -190,6 +191,7 @@ type CiliumNetworkingSpec struct { DisableK8sServices bool `json:"disableK8sServices,omitempty"` EnablePolicy string `json:"enablePolicy,omitempty"` EnableTracing bool `json:"enableTracing,omitempty"` + EnablePrometheusMetrics bool `json:"enablePrometheusMetrics,omitempty"` EnvoyLog string `json:"envoyLog,omitempty"` Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` Ipv4Node string `json:"ipv4Node,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/networking.go b/pkg/apis/kops/v1alpha1/networking.go index e24ebe6e4ff2e..b1be7d45dcf5e 100644 --- a/pkg/apis/kops/v1alpha1/networking.go +++ b/pkg/apis/kops/v1alpha1/networking.go @@ -174,6 +174,7 @@ type CiliumNetworkingSpec struct { AccessLog string `json:"accessLog,omitempty"` AgentLabels []string `json:"agentLabels,omitempty"` + AgentPrometheusPort int `json:"agentPrometheusPort,omitempty"` AllowLocalhost string `json:"allowLocalhost,omitempty"` AutoIpv6NodeRoutes bool `json:"autoIpv6NodeRoutes,omitempty"` BPFRoot string `json:"bpfRoot,omitempty"` @@ -186,6 +187,7 @@ type CiliumNetworkingSpec struct { DisableIpv4 bool `json:"disableIpv4,omitempty"` DisableK8sServices bool `json:"disableK8sServices,omitempty"` EnablePolicy string `json:"enablePolicy,omitempty"` + EnablePrometheusMetrics bool `json:"enablePrometheusMetrics,omitempty"` EnableTracing bool `json:"enableTracing,omitempty"` EnvoyLog string `json:"envoyLog,omitempty"` Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` diff --git a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go index af3f67eb6e362..db091b2c00bc5 100644 --- a/pkg/apis/kops/v1alpha1/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha1/zz_generated.conversion.go @@ -1220,6 +1220,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1232,6 +1233,7 @@ func autoConvert_v1alpha1_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.DisableIpv4 = in.DisableIpv4 out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnableTracing = in.EnableTracing out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize @@ -1296,6 +1298,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1309,6 +1312,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha1_CiliumNetworkingSpec(in * out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy out.EnableTracing = in.EnableTracing + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize out.Ipv4Node = in.Ipv4Node diff --git a/pkg/apis/kops/v1alpha2/networking.go b/pkg/apis/kops/v1alpha2/networking.go index 40b24086ce619..ab47c1a7cc698 100644 --- a/pkg/apis/kops/v1alpha2/networking.go +++ b/pkg/apis/kops/v1alpha2/networking.go @@ -175,6 +175,7 @@ type CiliumNetworkingSpec struct { AccessLog string `json:"accessLog,omitempty"` AgentLabels []string `json:"agentLabels,omitempty"` + AgentPrometheusPort int `json:"agentPrometheusPort,omitempty"` AllowLocalhost string `json:"allowLocalhost,omitempty"` AutoIpv6NodeRoutes bool `json:"autoIpv6NodeRoutes,omitempty"` BPFRoot string `json:"bpfRoot,omitempty"` @@ -187,6 +188,7 @@ type CiliumNetworkingSpec struct { DisableIpv4 bool `json:"disableIpv4,omitempty"` DisableK8sServices bool `json:"disableK8sServices,omitempty"` EnablePolicy string `json:"enablePolicy,omitempty"` + EnablePrometheusMetrics bool `json:"enablePrometheusMetrics,omitempty"` EnableTracing bool `json:"enableTracing,omitempty"` EnvoyLog string `json:"envoyLog,omitempty"` Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"` diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go index 5c2aef2addc19..c2ad36227d719 100644 --- a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go +++ b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go @@ -1262,6 +1262,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1274,6 +1275,7 @@ func autoConvert_v1alpha2_CiliumNetworkingSpec_To_kops_CiliumNetworkingSpec(in * out.DisableIpv4 = in.DisableIpv4 out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnableTracing = in.EnableTracing out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize @@ -1338,6 +1340,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.Version = in.Version out.AccessLog = in.AccessLog out.AgentLabels = in.AgentLabels + out.AgentPrometheusPort = in.AgentPrometheusPort out.AllowLocalhost = in.AllowLocalhost out.AutoIpv6NodeRoutes = in.AutoIpv6NodeRoutes out.BPFRoot = in.BPFRoot @@ -1351,6 +1354,7 @@ func autoConvert_kops_CiliumNetworkingSpec_To_v1alpha2_CiliumNetworkingSpec(in * out.DisableK8sServices = in.DisableK8sServices out.EnablePolicy = in.EnablePolicy out.EnableTracing = in.EnableTracing + out.EnablePrometheusMetrics = in.EnablePrometheusMetrics out.EnvoyLog = in.EnvoyLog out.Ipv4ClusterCIDRMaskSize = in.Ipv4ClusterCIDRMaskSize out.Ipv4Node = in.Ipv4Node diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template index bba3b264a87b9..c0d3d3667e1fb 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.12.yaml.template @@ -20,6 +20,15 @@ data: identity-allocation-mode: crd # If you want to run cilium in debug mode change this value to true debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}" + {{ if .EnablePrometheusMetrics }} + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}" + {{ end }} # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "{{- if or (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}" @@ -385,6 +394,13 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: cilium-agent + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: {{ or .AgentPrometheusPort "9090" }} + hostPort: {{ or .AgentPrometheusPort "9090" }} + name: prometheus + protocol: TCP + {{ end }} readinessProbe: exec: command: @@ -543,6 +559,10 @@ spec: - args: - --debug=$(CILIUM_DEBUG) - --identity-allocation-mode=$(CILIUM_IDENTITY_ALLOCATION_MODE) +{{ with .Networking.Cilium }} + {{ if .EnablePrometheusMetrics }} + - --enable-metrics + {{ end }} command: - cilium-operator env: @@ -622,12 +642,17 @@ spec: key: AWS_DEFAULT_REGION name: cilium-aws optional: true - -{{ with .Networking.Cilium }} image: "docker.io/cilium/operator:{{ .Version }}" -{{ end }} imagePullPolicy: IfNotPresent name: cilium-operator + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: 6942 + hostPort: 6942 + name: prometheus + protocol: TCP + {{ end }} +{{ end }} livenessProbe: httpGet: path: /healthz diff --git a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template index d7cc314a0a4f6..aa8b382961509 100644 --- a/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template +++ b/upup/models/cloudup/resources/addons/networking.cilium.io/k8s-1.7.yaml.template @@ -20,6 +20,15 @@ data: identity-allocation-mode: crd # If you want to run cilium in debug mode change this value to true debug: "{{- if .Debug -}}true{{- else -}}false{{- end -}}" + {{ if .EnablePrometheusMetrics }} + # If you want metrics enabled in all of your Cilium agents, set the port for + # which the Cilium agents will have their metrics exposed. + # This option deprecates the "prometheus-serve-addr" in the + # "cilium-metrics-config" ConfigMap + # NOTE that this will open the port on ALL nodes where Cilium pods are + # scheduled. + prometheus-serve-addr: ":{{- or .AgentPrometheusPort "9090" }}" + {{ end }} # Enable IPv4 addressing. If enabled, all endpoints are allocated an IPv4 # address. enable-ipv4: "{{- if or (.EnableIpv4) (and (not (.EnableIpv4)) (not (.EnableIpv6))) -}}true{{- else -}}false{{- end -}}" @@ -385,6 +394,13 @@ spec: successThreshold: 1 timeoutSeconds: 5 name: cilium-agent + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: {{ or .AgentPrometheusPort "9090" }} + hostPort: {{ or .AgentPrometheusPort "9090" }} + name: prometheus + protocol: TCP + {{ end }} readinessProbe: exec: command: @@ -541,6 +557,10 @@ spec: containers: - args: - --debug=$(CILIUM_DEBUG) +{{ with .Networking.Cilium }} + {{ if .EnablePrometheusMetrics }} + - --enable-metrics + {{ end }} command: - cilium-operator env: @@ -614,12 +634,17 @@ spec: key: AWS_DEFAULT_REGION name: cilium-aws optional: true - -{{ with .Networking.Cilium }} image: "docker.io/cilium/operator:{{ .Version }}" -{{ end }} imagePullPolicy: IfNotPresent name: cilium-operator + {{ if .EnablePrometheusMetrics }} + ports: + - containerPort: 6942 + hostPort: 6942 + name: prometheus + protocol: TCP + {{ end }} +{{ end }} livenessProbe: httpGet: path: /healthz