diff --git a/chart/kube-arangodb/templates/deployment-operator/role.yaml b/chart/kube-arangodb/templates/deployment-operator/role.yaml index fb1ce3c4e..e085b8316 100644 --- a/chart/kube-arangodb/templates/deployment-operator/role.yaml +++ b/chart/kube-arangodb/templates/deployment-operator/role.yaml @@ -30,7 +30,7 @@ rules: verbs: ["get", "list", "watch"] - apiGroups: ["monitoring.coreos.com"] resources: ["servicemonitors"] - verbs: ["get", "create", "delete"] + verbs: ["get", "create", "delete", "update"] {{- end }} {{- end }} \ No newline at end of file diff --git a/chart/kube-arangodb/values.yaml b/chart/kube-arangodb/values.yaml index 968faa1a5..1f292a4c6 100644 --- a/chart/kube-arangodb/values.yaml +++ b/chart/kube-arangodb/values.yaml @@ -33,7 +33,7 @@ operator: images: base: alpine:3.11 - metricsExporter: arangodb/arangodb-exporter:0.1.6 + metricsExporter: arangodb/arangodb-exporter:0.1.7 arango: arangodb/arangodb:latest rbac: enabled: true \ No newline at end of file diff --git a/pkg/apis/deployment/v1/metrics_spec.go b/pkg/apis/deployment/v1/metrics_spec.go index 4c47be9fd..d252cd5ff 100644 --- a/pkg/apis/deployment/v1/metrics_spec.go +++ b/pkg/apis/deployment/v1/metrics_spec.go @@ -17,6 +17,7 @@ // // Copyright holder is ArangoDB GmbH, Cologne, Germany // +// Adam Janikowski // package v1 @@ -33,12 +34,54 @@ type MetricsAuthenticationSpec struct { JWTTokenSecretName *string `json:"jwtTokenSecretName,omitempty"` } +// MetricsMode defines mode for metrics exporter +type MetricsMode string + +func (m MetricsMode) New() *MetricsMode { + return &m +} + +func (m MetricsMode) GetMetricsEndpoint() string { + switch m { + case MetricsModeInternal: + return k8sutil.ArangoExporterInternalEndpoint + default: + return k8sutil.ArangoExporterDefaultEndpoint + } +} + +const ( + // MetricsModeExporter exporter mode for old exporter type + MetricsModeExporter MetricsMode = "exporter" + MetricsModeSidecar MetricsMode = "sidecar" + MetricsModeInternal MetricsMode = "internal" +) + +func (m *MetricsMode) Get() MetricsMode { + if m == nil { + return MetricsModeExporter + } + + return *m +} + // MetricsSpec contains spec for arangodb exporter type MetricsSpec struct { Enabled *bool `json:"enabled,omitempty"` Image *string `json:"image,omitempty"` Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"` Resources v1.ResourceRequirements `json:"resources,omitempty"` + Mode *MetricsMode `json:"mode,omitempty"` + + Port *uint16 `json:"port,omitempty"` +} + +func (s *MetricsSpec) GetPort() uint16 { + if s == nil || s.Port == nil { + return k8sutil.ArangoExporterPort + } + + return *s.Port } // IsEnabled returns whether metrics are enabled or not diff --git a/pkg/deployment/deployment_metrics_test.go b/pkg/deployment/deployment_metrics_test.go new file mode 100644 index 000000000..72ce29446 --- /dev/null +++ b/pkg/deployment/deployment_metrics_test.go @@ -0,0 +1,435 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package deployment + +import ( + "testing" + + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + + "github.com/arangodb/kube-arangodb/pkg/util" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + core "k8s.io/api/core/v1" +) + +func TestEnsurePod_Metrics(t *testing.T) { + testCases := []testCaseStruct{ + { + Name: "DBserver Pod with metrics exporter and port override", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: func() api.MetricsSpec { + m := metricsSpec.DeepCopy() + + m.Port = util.NewUInt16(9999) + + return *m + }(), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + testCreateExporterContainerWithPort(false, emptyResources, 9999), + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter with mode", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: func() api.MetricsSpec { + m := metricsSpec.DeepCopy() + + m.Mode = api.MetricsModeExporter.New() + + return *m + }(), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + testCreateExporterContainer(false, emptyResources), + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter with internal mode", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: func() api.MetricsSpec { + m := metricsSpec.DeepCopy() + + m.Mode = api.MetricsModeInternal.New() + + return *m + }(), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: func() []core.ContainerPort { + ports := createTestPorts() + + ports = append(ports, core.ContainerPort{ + Name: "exporter", + Protocol: core.ProtocolTCP, + ContainerPort: k8sutil.ArangoPort, + }) + + return ports + }(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with metrics exporter with internal mode", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: func() api.MetricsSpec { + m := metricsSpec.DeepCopy() + + m.Mode = api.MetricsModeInternal.New() + + return *m + }(), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes + }, + ExpectedEvent: "member agent is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: func() []core.ContainerPort { + ports := createTestPorts() + + ports = append(ports, core.ContainerPort{ + Name: "exporter", + Protocol: core.ProtocolTCP, + ContainerPort: k8sutil.ArangoPort, + }) + + return ports + }(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with sidecar metrics exporter and port override", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: func() api.MetricsSpec { + m := metricsSpec.DeepCopy() + + m.Port = util.NewUInt16(9999) + + m.Mode = api.MetricsModeSidecar.New() + + return *m + }(), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + func() core.Container { + z := testCreateExporterContainerWithPort(false, emptyResources, 9999) + z.Command = append(z.Command, "--mode=passthru") + return z + }(), + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "Agency Pod with sidecar metrics exporter and port override", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: func() api.MetricsSpec { + m := metricsSpec.DeepCopy() + + m.Port = util.NewUInt16(9999) + + m.Mode = api.MetricsModeSidecar.New() + + return *m + }(), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = testYes + }, + ExpectedEvent: "member agent is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + func() core.Container { + z := testCreateExporterContainerWithPort(false, emptyResources, 9999) + z.Command = append(z.Command, "--mode=passthru") + return z + }(), + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + } + + runTestCases(t, testCases...) +} diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index f5e6790dd..1178dc2db 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -23,6 +23,7 @@ package deployment import ( + "fmt" "io/ioutil" "os" "testing" @@ -401,17 +402,17 @@ func createTestImages(enterprise bool) api.ImageInfoList { } } -func createTestExporterPorts() []core.ContainerPort { +func createTestExporterPorts(port uint16) []core.ContainerPort { return []core.ContainerPort{ { Name: "exporter", - ContainerPort: 9101, + ContainerPort: int32(port), Protocol: "TCP", }, } } -func createTestExporterCommand(secure bool) []string { +func createTestExporterCommand(secure bool, port uint16) []string { command := []string{ "/app/arangodb-exporter", } @@ -427,6 +428,11 @@ func createTestExporterCommand(secure bool) []string { if secure { command = append(command, "--ssl.keyfile=/secrets/tls/tls.keyfile") } + + if port != k8sutil.ArangoExporterPort { + command = append(command, fmt.Sprintf("--server.address=:%d", port)) + } + return command } @@ -480,14 +486,14 @@ func (testCase *testCaseStruct) createTestPodData(deployment *Deployment, group testCase.ExpectedPod.Spec.Tolerations = deployment.resources.CreatePodTolerations(group, groupSpec) } -func testCreateExporterContainer(secure bool, resources core.ResourceRequirements) core.Container { +func testCreateExporterContainerWithPort(secure bool, resources core.ResourceRequirements, port uint16) core.Container { var securityContext api.ServerGroupSpecSecurityContext return core.Container{ Name: k8sutil.ExporterContainerName, Image: testExporterImage, - Command: createTestExporterCommand(secure), - Ports: createTestExporterPorts(), + Command: createTestExporterCommand(secure, port), + Ports: createTestExporterPorts(port), VolumeMounts: []core.VolumeMount{ k8sutil.ExporterJWTVolumeMount(), }, @@ -497,3 +503,7 @@ func testCreateExporterContainer(secure bool, resources core.ResourceRequirement SecurityContext: securityContext.NewSecurityContext(), } } + +func testCreateExporterContainer(secure bool, resources core.ResourceRequirements) core.Container { + return testCreateExporterContainerWithPort(secure, resources, k8sutil.ArangoExporterPort) +} diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index e3c500575..35eb898bd 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -44,6 +44,7 @@ import ( ) var _ k8sutil.PodCreator = &ImageUpdatePod{} +var _ k8sutil.ContainerCreator = &ArangoDImageUpdateContainer{} type ImageUpdatePod struct { spec api.DeploymentSpec @@ -336,6 +337,16 @@ func (i *ImageUpdatePod) GetServiceAccountName() string { return "" } +func (a *ArangoDImageUpdateContainer) GetPorts() []core.ContainerPort { + return []core.ContainerPort{ + { + Name: "server", + ContainerPort: int32(k8sutil.ArangoPort), + Protocol: core.ProtocolTCP, + }, + } +} + func (a *ArangoDImageUpdateContainer) GetSecurityContext() *core.SecurityContext { // Default security context var v api.ServerGroupSpecSecurityContext diff --git a/pkg/deployment/resources/exporter.go b/pkg/deployment/resources/exporter.go index 04a2168f4..60d909c35 100644 --- a/pkg/deployment/resources/exporter.go +++ b/pkg/deployment/resources/exporter.go @@ -21,10 +21,13 @@ package resources import ( + "fmt" "path/filepath" "sort" "strconv" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "github.com/arangodb/kube-arangodb/pkg/util/constants" @@ -34,7 +37,8 @@ import ( // ArangodbExporterContainer creates metrics container func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sutil.HTTPProbeConfig, - resources v1.ResourceRequirements, securityContext *v1.SecurityContext) v1.Container { + resources v1.ResourceRequirements, securityContext *v1.SecurityContext, + spec api.DeploymentSpec) v1.Container { c := v1.Container{ Name: k8sutil.ExporterContainerName, @@ -43,7 +47,7 @@ func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sut Ports: []v1.ContainerPort{ { Name: "exporter", - ContainerPort: int32(k8sutil.ArangoExporterPort), + ContainerPort: int32(spec.Metrics.GetPort()), Protocol: v1.ProtocolTCP, }, }, @@ -59,11 +63,11 @@ func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sut return c } -func createExporterArgs(isSecure bool) []string { +func createExporterArgs(spec api.DeploymentSpec) []string { tokenpath := filepath.Join(k8sutil.ExporterJWTVolumeMountDir, constants.SecretKeyToken) options := make([]pod.OptionPair, 0, 64) scheme := "http" - if isSecure { + if spec.IsSecure() { scheme = "https" } options = append(options, @@ -71,11 +75,18 @@ func createExporterArgs(isSecure bool) []string { pod.OptionPair{"--arangodb.endpoint", scheme + "://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, ) keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) - if isSecure { + if spec.IsSecure() { options = append(options, pod.OptionPair{"--ssl.keyfile", keyPath}, ) } + + if port := spec.Metrics.GetPort(); port != k8sutil.ArangoExporterPort { + options = append(options, + pod.OptionPair{"--server.address", fmt.Sprintf(":%d", port)}, + ) + } + args := make([]string, 0, 2+len(options)) sort.Slice(options, func(i, j int) bool { return options[i].CompareTo(options[j]) < 0 diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index 8703b1ce7..532246895 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -43,6 +43,7 @@ const ( ) var _ k8sutil.PodCreator = &MemberArangoDPod{} +var _ k8sutil.ContainerCreator = &ArangoDContainer{} type MemberArangoDPod struct { status api.MemberStatus @@ -65,6 +66,29 @@ type ArangoDContainer struct { imageInfo api.ImageInfo } +func (a *ArangoDContainer) GetPorts() []core.ContainerPort { + ports := []core.ContainerPort{ + { + Name: "server", + ContainerPort: int32(k8sutil.ArangoPort), + Protocol: core.ProtocolTCP, + }, + } + + if a.spec.Metrics.IsEnabled() { + switch a.spec.Metrics.Mode.Get() { + case api.MetricsModeInternal: + ports = append(ports, core.ContainerPort{ + Name: "exporter", + ContainerPort: int32(k8sutil.ArangoPort), + Protocol: core.ProtocolTCP, + }) + } + } + + return ports +} + func (a *ArangoDContainer) GetExecutor() string { return ArangoDExecutor } @@ -215,26 +239,26 @@ func (m *MemberArangoDPod) GetServiceAccountName() string { func (m *MemberArangoDPod) GetSidecars(pod *core.Pod) { - if isMetricsEnabledForGroup(m.spec, m.group) { - image := m.context.GetMetricsExporterImage() - if m.spec.Metrics.HasImage() { - image = m.spec.Metrics.GetImage() - } + if m.spec.Metrics.IsEnabled() { + var c *core.Container - c := ArangodbExporterContainer(image, createExporterArgs(m.spec.IsSecure()), - createExporterLivenessProbe(m.spec.IsSecure()), m.spec.Metrics.Resources, - m.groupSpec.SecurityContext.NewSecurityContext()) + switch m.spec.Metrics.Mode.Get() { + case api.MetricsModeExporter: + if !m.group.IsExportMetrics() { + break + } + fallthrough + case api.MetricsModeSidecar: + c = m.createMetricsExporterSidecar() - if m.spec.Metrics.GetJWTTokenSecretName() != "" { - c.VolumeMounts = append(c.VolumeMounts, k8sutil.ExporterJWTVolumeMount()) + pod.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + default: + pod.Labels[k8sutil.LabelKeyArangoExporter] = "yes" } - if m.tlsKeyfileSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) + if c != nil { + pod.Spec.Containers = append(pod.Spec.Containers, *c) } - - pod.Spec.Containers = append(pod.Spec.Containers, c) - pod.Labels[k8sutil.LabelKeyArangoExporter] = "yes" } // A sidecar provided by the user @@ -277,11 +301,19 @@ func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { volumeMounts = append(volumeMounts, k8sutil.RocksdbEncryptionVolumeMount()) } - if isMetricsEnabledForGroup(m.spec, m.group) { - token := m.spec.Metrics.GetJWTTokenSecretName() - if token != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token) - volumes = append(volumes, vol) + if m.spec.Metrics.IsEnabled() { + switch m.spec.Metrics.Mode.Get() { + case api.MetricsModeExporter: + if !m.group.IsExportMetrics() { + break + } + fallthrough + case api.MetricsModeSidecar: + token := m.spec.Metrics.GetJWTTokenSecretName() + if token != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token) + volumes = append(volumes, vol) + } } } @@ -354,6 +386,33 @@ func (m *MemberArangoDPod) GetContainerCreator() k8sutil.ContainerCreator { } } -func isMetricsEnabledForGroup(spec api.DeploymentSpec, group api.ServerGroup) bool { - return spec.Metrics.IsEnabled() && group.IsExportMetrics() +func (m *MemberArangoDPod) isMetricsEnabledForGroup() bool { + return m.spec.Metrics.IsEnabled() && m.group.IsExportMetrics() +} + +func (m *MemberArangoDPod) createMetricsExporterSidecar() *core.Container { + image := m.context.GetMetricsExporterImage() + if m.spec.Metrics.HasImage() { + image = m.spec.Metrics.GetImage() + } + + args := createExporterArgs(m.spec) + if m.spec.Metrics.Mode.Get() == api.MetricsModeSidecar { + args = append(args, "--mode=passthru") + } + + c := ArangodbExporterContainer(image, args, + createExporterLivenessProbe(m.spec.IsSecure()), m.spec.Metrics.Resources, + m.groupSpec.SecurityContext.NewSecurityContext(), + m.spec) + + if m.spec.Metrics.GetJWTTokenSecretName() != "" { + c.VolumeMounts = append(c.VolumeMounts, k8sutil.ExporterJWTVolumeMount()) + } + + if m.tlsKeyfileSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) + } + + return &c } diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 3db0486c9..20a1405ff 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -47,6 +47,7 @@ type ArangoSyncContainer struct { } var _ k8sutil.PodCreator = &MemberSyncPod{} +var _ k8sutil.ContainerCreator = &ArangoSyncContainer{} type MemberSyncPod struct { tlsKeyfileSecretName string @@ -60,6 +61,16 @@ type MemberSyncPod struct { imageInfo api.ImageInfo } +func (a *ArangoSyncContainer) GetPorts() []core.ContainerPort { + return []core.ContainerPort{ + { + Name: "server", + ContainerPort: int32(k8sutil.ArangoPort), + Protocol: core.ProtocolTCP, + }, + } +} + func (a *ArangoSyncContainer) GetExecutor() string { return ArangoSyncExecutor } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 9bf9c2667..dbab439af 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -27,6 +27,11 @@ import ( "encoding/hex" "time" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + jg "github.com/dgrijalva/jwt-go" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -116,32 +121,74 @@ func (r *Resources) ensureTokenSecret(secrets k8sutil.SecretInterface, secretNam return nil } +var ( + exporterTokenClaims = map[string]interface{}{ + "iss": "arangodb", + "server_id": "exporter", + "allowed_paths": []string{"/_admin/statistics", "/_admin/statistics-description", k8sutil.ArangoExporterInternalEndpoint}, + } +) + // ensureExporterTokenSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with correct access. func (r *Resources) ensureExporterTokenSecret(secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) error { - if _, err := secrets.Get(tokenSecretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) { - // Secret not found, create it - claims := map[string]interface{}{ - "iss": "arangodb", - "server_id": "exporter", - "allowed_paths": []string{"/_admin/statistics", "/_admin/statistics-description"}, - } + if recreate, exists, err := r.ensureExporterTokenSecretCreateRequired(secrets, tokenSecretName, secretSecretName); err != nil { + return err + } else if recreate { // Create secret + if exists { + if err := secrets.Delete(tokenSecretName, nil); err != nil && !apierrors.IsNotFound(err) { + return err + } + } + owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateJWTFromSecret(secrets, tokenSecretName, secretSecretName, claims, &owner); k8sutil.IsAlreadyExists(err) { + if err := k8sutil.CreateJWTFromSecret(secrets, tokenSecretName, secretSecretName, exporterTokenClaims, &owner); k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { // Failed to create secret return maskAny(err) } - } else if err != nil { - // Failed to get secret for other reasons - return maskAny(err) } return nil } +func (r *Resources) ensureExporterTokenSecretCreateRequired(secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) (bool, bool, error) { + if secret, err := secrets.Get(tokenSecretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) { + return true, false, nil + } else if err == nil { + // Check if claims are fine + data, ok := secret.Data[constants.SecretKeyToken] + if !ok { + return true, true, nil + } + + secret, err := k8sutil.GetTokenSecret(secrets, secretSecretName) + if err != nil { + return false, true, maskAny(err) + } + + token, err := jg.Parse(string(data), func(token *jg.Token) (i interface{}, err error) { + return []byte(secret), nil + }) + + if err != nil { + return true, true, nil + } + + tokenClaims, ok := token.Claims.(jg.MapClaims) + if !ok { + return true, true, nil + } + + return !equality.Semantic.DeepEqual(tokenClaims, exporterTokenClaims), true, nil + } else { + // Failed to get secret for other reasons + return false, false, maskAny(err) + } +} + // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. func (r *Resources) ensureTLSCACertificateSecret(secrets k8sutil.SecretInterface, spec api.TLSSpec) error { diff --git a/pkg/deployment/resources/servicemonitor.go b/pkg/deployment/resources/servicemonitor.go index 3cac64acd..4368d92ac 100644 --- a/pkg/deployment/resources/servicemonitor.go +++ b/pkg/deployment/resources/servicemonitor.go @@ -24,7 +24,12 @@ package resources import ( "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + deploymentApi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "k8s.io/apimachinery/pkg/api/equality" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" coreosv1 "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" clientv1 "github.com/coreos/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" @@ -88,6 +93,45 @@ func (r *Resources) makeEndpoint(isSecure bool) coreosv1.Endpoint { } } +func (r *Resources) serviceMonitorSpec() (coreosv1.ServiceMonitorSpec, error) { + apiObject := r.context.GetAPIObject() + deploymentName := apiObject.GetName() + spec := r.context.GetSpec() + + switch spec.Metrics.Mode.Get() { + case deploymentApi.MetricsModeInternal: + if spec.Metrics.Authentication.JWTTokenSecretName == nil { + return coreosv1.ServiceMonitorSpec{}, errors.NewNotFound(schema.GroupResource{Group: "v1/secret"}, "metrics-secret") + } + + endpoint := r.makeEndpoint(spec.IsSecure()) + + endpoint.BearerTokenSecret.Name = *spec.Metrics.Authentication.JWTTokenSecretName + endpoint.BearerTokenSecret.Key = constants.SecretKeyToken + endpoint.Path = k8sutil.ArangoExporterInternalEndpoint + + return coreosv1.ServiceMonitorSpec{ + JobLabel: "k8s-app", + Endpoints: []coreosv1.Endpoint{ + endpoint, + }, + Selector: metav1.LabelSelector{ + MatchLabels: LabelsForExporterServiceMonitorSelector(deploymentName), + }, + }, nil + default: + return coreosv1.ServiceMonitorSpec{ + JobLabel: "k8s-app", + Endpoints: []coreosv1.Endpoint{ + r.makeEndpoint(spec.IsSecure()), + }, + Selector: metav1.LabelSelector{ + MatchLabels: LabelsForExporterServiceMonitorSelector(deploymentName), + }, + }, nil + } +} + // EnsureServiceMonitor creates or updates a ServiceMonitor. func (r *Resources) EnsureServiceMonitor() error { // Some preparations: @@ -114,6 +158,12 @@ func (r *Resources) EnsureServiceMonitor() error { if !wantMetrics { return nil } + + spec, err := r.serviceMonitorSpec() + if err != nil { + return err + } + // Need to create one: smon := &coreosv1.ServiceMonitor{ ObjectMeta: metav1.ObjectMeta{ @@ -121,15 +171,7 @@ func (r *Resources) EnsureServiceMonitor() error { Labels: LabelsForExporterServiceMonitor(deploymentName), OwnerReferences: []metav1.OwnerReference{owner}, }, - Spec: coreosv1.ServiceMonitorSpec{ - JobLabel: "k8s-app", - Endpoints: []coreosv1.Endpoint{ - r.makeEndpoint(spec.IsSecure()), - }, - Selector: metav1.LabelSelector{ - MatchLabels: LabelsForExporterServiceMonitorSelector(deploymentName), - }, - }, + Spec: spec, } smon, err = serviceMonitors.Create(smon) if err != nil { @@ -143,11 +185,6 @@ func (r *Resources) EnsureServiceMonitor() error { return maskAny(err) } } - if wantMetrics { - log.Debug().Msgf("ServiceMonitor %s already found, no need to create.", - serviceMonitorName) - return nil - } // Check if the service monitor is ours, otherwise we do not touch it: found := false for _, owner := range servMon.ObjectMeta.OwnerReferences { @@ -161,6 +198,30 @@ func (r *Resources) EnsureServiceMonitor() error { log.Debug().Msgf("Found unneeded ServiceMonitor %s, but not owned by us, will not touch it", serviceMonitorName) return nil } + if wantMetrics { + log.Debug().Msgf("ServiceMonitor %s already found, ensuring it is fine.", + serviceMonitorName) + + spec, err := r.serviceMonitorSpec() + if err != nil { + return err + } + + if equality.Semantic.DeepDerivative(spec, servMon.Spec) { + log.Debug().Msgf("ServiceMonitor %s already found and up to date.", + serviceMonitorName) + return nil + } + + servMon.Spec = spec + + _, err = serviceMonitors.Update(servMon) + if err != nil { + return err + } + + return nil + } // Need to get rid of the ServiceMonitor: err = serviceMonitors.Delete(serviceMonitorName, &metav1.DeleteOptions{}) if err == nil { diff --git a/pkg/util/k8sutil/constants.go b/pkg/util/k8sutil/constants.go index d0799caf0..07564c77e 100644 --- a/pkg/util/k8sutil/constants.go +++ b/pkg/util/k8sutil/constants.go @@ -29,6 +29,9 @@ const ( ArangoSyncWorkerPort = 8729 ArangoExporterPort = 9101 + ArangoExporterInternalEndpoint = "/_admin/metrics" + ArangoExporterDefaultEndpoint = "/metrics" + // K8s constants ClusterIPNone = "None" TopologyKeyHostname = "kubernetes.io/hostname" diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index bef12a67b..258825e21 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -86,6 +86,7 @@ type ContainerCreator interface { GetImage() string GetEnvs() []core.EnvVar GetSecurityContext() *core.SecurityContext + GetPorts() []core.ContainerPort } // IsPodReady returns true if the PodReady condition on @@ -344,16 +345,10 @@ func NewContainer(args []string, containerCreator ContainerCreator) (core.Contai } return core.Container{ - Name: ServerContainerName, - Image: containerCreator.GetImage(), - Command: append([]string{containerCreator.GetExecutor()}, args...), - Ports: []core.ContainerPort{ - { - Name: "server", - ContainerPort: int32(ArangoPort), - Protocol: core.ProtocolTCP, - }, - }, + Name: ServerContainerName, + Image: containerCreator.GetImage(), + Command: append([]string{containerCreator.GetExecutor()}, args...), + Ports: containerCreator.GetPorts(), Env: containerCreator.GetEnvs(), Resources: containerCreator.GetResourceRequirements(), LivenessProbe: liveness, diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 8431c2f2f..b8c2d5aea 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -36,6 +36,7 @@ import ( type SecretInterface interface { Create(*v1.Secret) (*v1.Secret, error) Get(name string, options metav1.GetOptions) (*v1.Secret, error) + Delete(name string, options *metav1.DeleteOptions) error } // ValidateEncryptionKeySecret checks that a secret with given name in given namespace diff --git a/pkg/util/k8sutil/secrets_cache.go b/pkg/util/k8sutil/secrets_cache.go index ee866d78e..40775c2b5 100644 --- a/pkg/util/k8sutil/secrets_cache.go +++ b/pkg/util/k8sutil/secrets_cache.go @@ -58,6 +58,15 @@ func (sc *secretsCache) Create(s *v1.Secret) (*v1.Secret, error) { return result, nil } +func (sc *secretsCache) Delete(name string, options *metav1.DeleteOptions) error { + sc.cache = nil + err := sc.cli.Delete(name, options) + if err != nil { + return maskAny(err) + } + return nil +} + func (sc *secretsCache) Get(name string, options metav1.GetOptions) (*v1.Secret, error) { if sc.cache == nil { list, err := sc.cli.List(metav1.ListOptions{}) diff --git a/pkg/util/refs.go b/pkg/util/refs.go index f9b41be31..dd1c4142f 100644 --- a/pkg/util/refs.go +++ b/pkg/util/refs.go @@ -100,6 +100,30 @@ func Int32OrDefault(input *int32, defaultValue ...int32) int32 { return *input } +// NewUInt16 returns a reference to an uint16 with given value. +func NewUInt16(input uint16) *uint16 { + return &input +} + +// NewUInt16OrNil returns nil if input is nil, otherwise returns a clone of the given value. +func NewUInt16OrNil(input *uint16) *uint16 { + if input == nil { + return nil + } + return NewUInt16(*input) +} + +// UInt16OrDefault returns the default value (or 0) if input is nil, otherwise returns the referenced value. +func UInt16OrDefault(input *uint16, defaultValue ...uint16) uint16 { + if input == nil { + if len(defaultValue) > 0 { + return defaultValue[0] + } + return 0 + } + return *input +} + // NewBool returns a reference to a bool with given value. func NewBool(input bool) *bool { return &input