diff --git a/pkg/apis/deployment/v1/deployment.go b/pkg/apis/deployment/v1/deployment.go index 5be4092a5..e7952837b 100644 --- a/pkg/apis/deployment/v1/deployment.go +++ b/pkg/apis/deployment/v1/deployment.go @@ -49,6 +49,8 @@ type ArangoDeployment struct { Status DeploymentStatus `json:"status"` } +type ServerGroupFunc func(ServerGroup, ServerGroupSpec, *MemberStatusList) error + // AsOwner creates an OwnerReference for the given deployment func (d *ArangoDeployment) AsOwner() metav1.OwnerReference { trueVar := true @@ -67,7 +69,7 @@ func (d *ArangoDeployment) AsOwner() metav1.OwnerReference { // If the callback returns an error, this error is returned and no other server // groups are processed. // Groups are processed in this order: agents, single, dbservers, coordinators, syncmasters, syncworkers -func (d *ArangoDeployment) ForeachServerGroup(cb func(group ServerGroup, spec ServerGroupSpec, status *MemberStatusList) error, status *DeploymentStatus) error { +func (d *ArangoDeployment) ForeachServerGroup(cb ServerGroupFunc, status *DeploymentStatus) error { if status == nil { status = &d.Status } diff --git a/pkg/apis/deployment/v1/deployment_spec.go b/pkg/apis/deployment/v1/deployment_spec.go index af8d5d80d..7dda7260c 100644 --- a/pkg/apis/deployment/v1/deployment_spec.go +++ b/pkg/apis/deployment/v1/deployment_spec.go @@ -70,6 +70,7 @@ type DeploymentSpec struct { Sync SyncSpec `json:"sync"` License LicenseSpec `json:"license"` Metrics MetricsSpec `json:"metrics"` + Lifecycle LifecycleSpec `json:"lifecycle,omitempty"` Single ServerGroupSpec `json:"single"` Agents ServerGroupSpec `json:"agents"` @@ -278,6 +279,7 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) { s.SyncMasters.SetDefaultsFrom(source.SyncMasters) s.SyncWorkers.SetDefaultsFrom(source.SyncWorkers) s.Metrics.SetDefaultsFrom(source.Metrics) + s.Lifecycle.SetDefaultsFrom(source.Lifecycle) s.Chaos.SetDefaultsFrom(source.Chaos) s.Bootstrap.SetDefaultsFrom(source.Bootstrap) } diff --git a/pkg/apis/deployment/v1/lifecycle_spec.go b/pkg/apis/deployment/v1/lifecycle_spec.go new file mode 100644 index 000000000..0bbd3d70b --- /dev/null +++ b/pkg/apis/deployment/v1/lifecycle_spec.go @@ -0,0 +1,37 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + +package v1 + +import ( + v1 "k8s.io/api/core/v1" +) + +type LifecycleSpec struct { + Resources v1.ResourceRequirements `json:"resources,omitempty"` +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *LifecycleSpec) SetDefaultsFrom(source LifecycleSpec) { + setDefaultsFromResourceList(&s.Resources.Limits, source.Resources.Limits) + setDefaultsFromResourceList(&s.Resources.Requests, source.Resources.Requests) +} diff --git a/pkg/apis/deployment/v1/metrics_spec.go b/pkg/apis/deployment/v1/metrics_spec.go index 313b507c2..07166d1c4 100644 --- a/pkg/apis/deployment/v1/metrics_spec.go +++ b/pkg/apis/deployment/v1/metrics_spec.go @@ -24,6 +24,7 @@ package v1 import ( "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" ) // MetricsAuthenticationSpec contains spec for authentication with arangodb @@ -37,6 +38,7 @@ type MetricsSpec struct { Enabled *bool `json:"enabled,omitempty"` Image *string `json:"image,omitempty"` Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"` + Resources v1.ResourceRequirements `json:"resources,omitempty"` } // IsEnabled returns whether metrics are enabled or not @@ -85,6 +87,8 @@ func (s *MetricsSpec) SetDefaultsFrom(source MetricsSpec) { if s.Authentication.JWTTokenSecretName == nil { s.Authentication.JWTTokenSecretName = util.NewStringOrNil(source.Authentication.JWTTokenSecretName) } + setDefaultsFromResourceList(&s.Resources.Limits, source.Resources.Limits) + setDefaultsFromResourceList(&s.Resources.Requests, source.Resources.Requests) } // Validate the given spec diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index b3caa83fd..56d49dad5 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -312,6 +312,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.Sync.DeepCopyInto(&out.Sync) in.License.DeepCopyInto(&out.License) in.Metrics.DeepCopyInto(&out.Metrics) + in.Lifecycle.DeepCopyInto(&out.Lifecycle) in.Single.DeepCopyInto(&out.Single) in.Agents.DeepCopyInto(&out.Agents) in.DBServers.DeepCopyInto(&out.DBServers) @@ -550,6 +551,23 @@ func (in *LicenseSpec) DeepCopy() *LicenseSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LifecycleSpec) DeepCopyInto(out *LifecycleSpec) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LifecycleSpec. +func (in *LifecycleSpec) DeepCopy() *LifecycleSpec { + if in == nil { + return nil + } + out := new(LifecycleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in @@ -645,6 +663,7 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { **out = **in } in.Authentication.DeepCopyInto(&out.Authentication) + in.Resources.DeepCopyInto(&out.Resources) return } diff --git a/pkg/apis/deployment/v1alpha/deployment.go b/pkg/apis/deployment/v1alpha/deployment.go index 92fe09ebd..27b22e363 100644 --- a/pkg/apis/deployment/v1alpha/deployment.go +++ b/pkg/apis/deployment/v1alpha/deployment.go @@ -49,6 +49,8 @@ type ArangoDeployment struct { Status DeploymentStatus `json:"status"` } +type ServerGroupFunc func(ServerGroup, ServerGroupSpec, *MemberStatusList) error + // AsOwner creates an OwnerReference for the given deployment func (d *ArangoDeployment) AsOwner() metav1.OwnerReference { trueVar := true @@ -67,7 +69,7 @@ func (d *ArangoDeployment) AsOwner() metav1.OwnerReference { // If the callback returns an error, this error is returned and no other server // groups are processed. // Groups are processed in this order: agents, single, dbservers, coordinators, syncmasters, syncworkers -func (d *ArangoDeployment) ForeachServerGroup(cb func(group ServerGroup, spec ServerGroupSpec, status *MemberStatusList) error, status *DeploymentStatus) error { +func (d *ArangoDeployment) ForeachServerGroup(cb ServerGroupFunc, status *DeploymentStatus) error { if status == nil { status = &d.Status } diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index 7e7cf162f..4db326af3 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -70,6 +70,7 @@ type DeploymentSpec struct { Sync SyncSpec `json:"sync"` License LicenseSpec `json:"license"` Metrics MetricsSpec `json:"metrics"` + Lifecycle LifecycleSpec `json:"lifecycle,omitempty"` Single ServerGroupSpec `json:"single"` Agents ServerGroupSpec `json:"agents"` @@ -278,6 +279,7 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) { s.SyncMasters.SetDefaultsFrom(source.SyncMasters) s.SyncWorkers.SetDefaultsFrom(source.SyncWorkers) s.Metrics.SetDefaultsFrom(source.Metrics) + s.Lifecycle.SetDefaultsFrom(source.Lifecycle) s.Chaos.SetDefaultsFrom(source.Chaos) s.Bootstrap.SetDefaultsFrom(source.Bootstrap) } diff --git a/pkg/apis/deployment/v1alpha/lifecycle_spec.go b/pkg/apis/deployment/v1alpha/lifecycle_spec.go new file mode 100644 index 000000000..c7513057d --- /dev/null +++ b/pkg/apis/deployment/v1alpha/lifecycle_spec.go @@ -0,0 +1,37 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + +package v1alpha + +import ( + v1 "k8s.io/api/core/v1" +) + +type LifecycleSpec struct { + Resources v1.ResourceRequirements `json:"resources,omitempty"` +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *LifecycleSpec) SetDefaultsFrom(source LifecycleSpec) { + setDefaultsFromResourceList(&s.Resources.Limits, source.Resources.Limits) + setDefaultsFromResourceList(&s.Resources.Requests, source.Resources.Requests) +} diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index 41c316bb5..2b9750d11 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -24,6 +24,7 @@ package v1alpha import ( "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" ) // MetricsAuthenticationSpec contains spec for authentication with arangodb @@ -37,6 +38,7 @@ type MetricsSpec struct { Enabled *bool `json:"enabled,omitempty"` Image *string `json:"image,omitempty"` Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"` + Resources v1.ResourceRequirements `json:"resources,omitempty"` } // IsEnabled returns whether metrics are enabled or not @@ -85,6 +87,8 @@ func (s *MetricsSpec) SetDefaultsFrom(source MetricsSpec) { if s.Authentication.JWTTokenSecretName == nil { s.Authentication.JWTTokenSecretName = util.NewStringOrNil(source.Authentication.JWTTokenSecretName) } + setDefaultsFromResourceList(&s.Resources.Limits, source.Resources.Limits) + setDefaultsFromResourceList(&s.Resources.Requests, source.Resources.Requests) } // Validate the given spec diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 1cd3518cc..d8234c456 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -312,6 +312,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.Sync.DeepCopyInto(&out.Sync) in.License.DeepCopyInto(&out.License) in.Metrics.DeepCopyInto(&out.Metrics) + in.Lifecycle.DeepCopyInto(&out.Lifecycle) in.Single.DeepCopyInto(&out.Single) in.Agents.DeepCopyInto(&out.Agents) in.DBServers.DeepCopyInto(&out.DBServers) @@ -550,6 +551,23 @@ func (in *LicenseSpec) DeepCopy() *LicenseSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *LifecycleSpec) DeepCopyInto(out *LifecycleSpec) { + *out = *in + in.Resources.DeepCopyInto(&out.Resources) + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LifecycleSpec. +func (in *LifecycleSpec) DeepCopy() *LifecycleSpec { + if in == nil { + return nil + } + out := new(LifecycleSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MemberStatus) DeepCopyInto(out *MemberStatus) { *out = *in @@ -645,6 +663,7 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { **out = **in } in.Authentication.DeepCopyInto(&out.Authentication) + in.Resources.DeepCopyInto(&out.Resources) return } diff --git a/pkg/deployment/deployment_test.go b/pkg/deployment/deployment_test.go index e2676a9a3..56d6f6f6a 100644 --- a/pkg/deployment/deployment_test.go +++ b/pkg/deployment/deployment_test.go @@ -70,6 +70,7 @@ const ( testServiceAccountName = "testServiceAccountName" testPriorityClassName = "testPriority" testImageLifecycle = "arangodb/kube-arangodb:0.3.16" + testExporterImage = "arangodb/arangodb-exporter:0.1.6" testImageAlpine = "alpine:3.7" ) @@ -149,7 +150,7 @@ func TestEnsurePods(t *testing.T) { metricsSpec := api.MetricsSpec{ Enabled: util.NewBool(true), - Image: util.NewString("arangodb/arangodb-exporter:0.1.6"), + Image: util.NewString(testExporterImage), Authentication: api.MetricsAuthenticationSpec{ JWTTokenSecretName: util.NewString(testExporterToken), }, @@ -168,6 +169,11 @@ func TestEnsurePods(t *testing.T) { }, } + emptyResources := v1.ResourceRequirements{ + Limits: make(v1.ResourceList), + Requests: make(v1.ResourceList), + } + sidecarName1 := "sidecar1" sidecarName2 := "sidecar2" @@ -1028,20 +1034,70 @@ func TestEnsurePods(t *testing.T) { Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, }, + testCreateExporterContainer(false, emptyResources), + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter which contains resource requirements", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: api.MetricsSpec{ + Enabled: util.NewBool(true), + Image: util.NewString(testExporterImage), + Authentication: api.MetricsAuthenticationSpec{ + JWTTokenSecretName: util.NewString(testExporterToken), + }, + Resources: resourcesUnfiltered, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + }, + Containers: []v1.Container{ { - Name: k8sutil.ExporterContainerName, - Image: "arangodb/arangodb-exporter:0.1.6", - Command: createTestExporterCommand(false), - Ports: createTestExporterPorts(), + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), VolumeMounts: []v1.VolumeMount{ - k8sutil.ExporterJWTVolumeMount(), + k8sutil.ArangodVolumeMount(), }, - LivenessProbe: createTestExporterLivenessProbe(false), + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), ImagePullPolicy: v1.PullIfNotPresent, SecurityContext: &v1.SecurityContext{ Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, }, + testCreateExporterContainer(false, k8sutil.ExtractPodResourceRequirement(resourcesUnfiltered)), }, RestartPolicy: v1.RestartPolicyNever, TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, @@ -1053,13 +1109,16 @@ func TestEnsurePods(t *testing.T) { }, }, { - Name: "DBserver Pod with metrics exporter and lifecycle init container", + Name: "DBserver Pod with lifecycle init container which contains resource requirements", ArangoDeployment: &api.ArangoDeployment{ Spec: api.DeploymentSpec{ Image: util.NewString(testImage), Authentication: noAuthentication, TLS: noTLS, Metrics: metricsSpec, + Lifecycle: api.LifecycleSpec{ + Resources: resourcesUnfiltered, + }, }, }, Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { @@ -1087,7 +1146,7 @@ func TestEnsurePods(t *testing.T) { k8sutil.LifecycleVolume(), }, InitContainers: []v1.Container{ - createTestLifecycleContainer(), + createTestLifecycleContainer(k8sutil.ExtractPodResourceRequirement(resourcesUnfiltered)), }, Containers: []v1.Container{ { @@ -1112,20 +1171,7 @@ func TestEnsurePods(t *testing.T) { Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, }, - { - Name: k8sutil.ExporterContainerName, - Image: "arangodb/arangodb-exporter:0.1.6", - Command: createTestExporterCommand(false), - Ports: createTestExporterPorts(), - VolumeMounts: []v1.VolumeMount{ - k8sutil.ExporterJWTVolumeMount(), - }, - LivenessProbe: createTestExporterLivenessProbe(false), - ImagePullPolicy: v1.PullIfNotPresent, - SecurityContext: &v1.SecurityContext{ - Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, - }, - }, + testCreateExporterContainer(false, emptyResources), }, RestartPolicy: v1.RestartPolicyNever, TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, @@ -1172,7 +1218,7 @@ func TestEnsurePods(t *testing.T) { k8sutil.LifecycleVolume(), }, InitContainers: []v1.Container{ - createTestLifecycleContainer(), + createTestLifecycleContainer(emptyResources), createTestAlpineContainer(firstDBServerStatus.ID, false), }, Containers: []v1.Container{ @@ -1198,20 +1244,7 @@ func TestEnsurePods(t *testing.T) { Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, }, - { - Name: k8sutil.ExporterContainerName, - Image: "arangodb/arangodb-exporter:0.1.6", - Command: createTestExporterCommand(false), - Ports: createTestExporterPorts(), - VolumeMounts: []v1.VolumeMount{ - k8sutil.ExporterJWTVolumeMount(), - }, - LivenessProbe: createTestExporterLivenessProbe(false), - ImagePullPolicy: v1.PullIfNotPresent, - SecurityContext: &v1.SecurityContext{ - Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, - }, - }, + testCreateExporterContainer(false, emptyResources), }, RestartPolicy: v1.RestartPolicyNever, TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, @@ -1259,6 +1292,8 @@ func TestEnsurePods(t *testing.T) { testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, authorization, k8sutil.ArangoPort) + testCase.ExpectedPod.Spec.Containers[1].VolumeMounts = append( + testCase.ExpectedPod.Spec.Containers[1].VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) }, config: Config{ LifecycleImage: testImageLifecycle, @@ -1275,7 +1310,7 @@ func TestEnsurePods(t *testing.T) { k8sutil.LifecycleVolume(), }, InitContainers: []v1.Container{ - createTestLifecycleContainer(), + createTestLifecycleContainer(emptyResources), }, Containers: []v1.Container{ { @@ -1305,21 +1340,7 @@ func TestEnsurePods(t *testing.T) { k8sutil.ClusterJWTVolumeMount(), }, }, - { - Name: k8sutil.ExporterContainerName, - Image: "arangodb/arangodb-exporter:0.1.6", - Command: createTestExporterCommand(true), - Ports: createTestExporterPorts(), - VolumeMounts: []v1.VolumeMount{ - k8sutil.ExporterJWTVolumeMount(), - k8sutil.TlsKeyfileVolumeMount(), - }, - LivenessProbe: createTestExporterLivenessProbe(true), - ImagePullPolicy: v1.PullIfNotPresent, - SecurityContext: &v1.SecurityContext{ - Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, - }, - }, + testCreateExporterContainer(true, emptyResources), }, RestartPolicy: v1.RestartPolicyNever, TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, @@ -1374,10 +1395,7 @@ func TestEnsurePods(t *testing.T) { Command: createTestCommandForCoordinator(firstCoordinatorStatus.ID, true, true, false), Ports: createTestPorts(), ImagePullPolicy: v1.PullIfNotPresent, - Resources: v1.ResourceRequirements{ - Limits: make(v1.ResourceList), - Requests: make(v1.ResourceList), - }, + Resources: emptyResources, SecurityContext: &v1.SecurityContext{ Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, @@ -1747,7 +1765,7 @@ func TestEnsurePods(t *testing.T) { testDeploymentName+"-sync-jwt"), }, InitContainers: []v1.Container{ - createTestLifecycleContainer(), + createTestLifecycleContainer(emptyResources), }, Containers: []v1.Container{ { @@ -1843,7 +1861,7 @@ func TestEnsurePods(t *testing.T) { k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, testDeploymentName+"-sync-jwt"), }, InitContainers: []v1.Container{ - createTestLifecycleContainer(), + createTestLifecycleContainer(emptyResources), }, Containers: []v1.Container{ { @@ -2307,7 +2325,7 @@ func createTestExporterLivenessProbe(secure bool) *v1.Probe { }.Create() } -func createTestLifecycleContainer() v1.Container { +func createTestLifecycleContainer(resources v1.ResourceRequirements) v1.Container { binaryPath, _ := os.Executable() return v1.Container{ @@ -2318,6 +2336,7 @@ func createTestLifecycleContainer() v1.Container { k8sutil.LifecycleVolumeMount(), }, ImagePullPolicy: "IfNotPresent", + Resources: resources, SecurityContext: &v1.SecurityContext{ Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, @@ -2347,3 +2366,21 @@ func (testCase *testCaseStruct) createTestPodData(deployment *Deployment, group groupSpec := testCase.ArangoDeployment.Spec.GetServerGroupSpec(group) testCase.ExpectedPod.Spec.Tolerations = deployment.resources.CreatePodTolerations(group, groupSpec) } + +func testCreateExporterContainer(secure bool, resources v1.ResourceRequirements) v1.Container { + return v1.Container{ + Name: k8sutil.ExporterContainerName, + Image: testExporterImage, + Command: createTestExporterCommand(secure), + Ports: createTestExporterPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ExporterJWTVolumeMount(), + }, + Resources: resources, + LivenessProbe: createTestExporterLivenessProbe(secure), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + } +} diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index ed3c82a6a..f30793856 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -23,14 +23,10 @@ package reconcile import ( - "reflect" - "strings" - driver "github.com/arangodb/go-driver" upgraderules "github.com/arangodb/go-upgrade-rules" "github.com/rs/zerolog" "github.com/rs/zerolog/log" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" @@ -135,93 +131,12 @@ func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, // Check for scale up/down if len(plan) == 0 { - switch spec.GetMode() { - case api.DeploymentModeSingle: - // Never scale down - case api.DeploymentModeActiveFailover: - // Only scale singles - plan = append(plan, createScalePlan(log, status.Members.Single, api.ServerGroupSingle, spec.Single.GetCount())...) - case api.DeploymentModeCluster: - // Scale dbservers, coordinators - plan = append(plan, createScalePlan(log, status.Members.DBServers, api.ServerGroupDBServers, spec.DBServers.GetCount())...) - plan = append(plan, createScalePlan(log, status.Members.Coordinators, api.ServerGroupCoordinators, spec.Coordinators.GetCount())...) - } - if spec.GetMode().SupportsSync() { - // Scale syncmasters & syncworkers - plan = append(plan, createScalePlan(log, status.Members.SyncMasters, api.ServerGroupSyncMasters, spec.SyncMasters.GetCount())...) - plan = append(plan, createScalePlan(log, status.Members.SyncWorkers, api.ServerGroupSyncWorkers, spec.SyncWorkers.GetCount())...) - } + plan = createScaleMemeberPlan(log, spec, status) } // Check for the need to rotate one or more members if len(plan) == 0 { - getPod := func(podName string) *v1.Pod { - for _, p := range pods { - if p.GetName() == podName { - return &p - } - } - return nil - } - // createRotateOrUpgradePlan goes over all pods to check if an upgrade or rotate - // is needed. If an upgrade is needed but not allowed, the second return value - // will be true. - // Returns: (newPlan, upgradeNotAllowed) - createRotateOrUpgradePlan := func() (api.Plan, bool, driver.Version, driver.Version, upgraderules.License, upgraderules.License) { - var newPlan api.Plan - upgradeNotAllowed := false - var fromVersion, toVersion driver.Version - var fromLicense, toLicense upgraderules.License - status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { - for _, m := range members { - if m.Phase != api.MemberPhaseCreated { - // Only rotate when phase is created - continue - } - if podName := m.PodName; podName != "" { - if p := getPod(podName); p != nil { - // Got pod, compare it with what it should be - decision := podNeedsUpgrading(log, *p, spec, status.Images) - if decision.UpgradeNeeded && !decision.UpgradeAllowed { - // Oops, upgrade is not allowed - upgradeNotAllowed = true - fromVersion = decision.FromVersion - fromLicense = decision.FromLicense - toVersion = decision.ToVersion - toLicense = decision.ToLicense - return nil - } else if len(newPlan) == 0 { - // Only rotate/upgrade 1 pod at a time - if decision.UpgradeNeeded { - // Yes, upgrade is needed (and allowed) - newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec.GetImage(), status, !decision.AutoUpgradeNeeded) - - } else { - // Upgrade is not needed, see if rotation is needed - if rotNeeded, reason := podNeedsRotation(log, *p, apiObject, spec, group, status, m.ID, context); rotNeeded { - newPlan = createRotateMemberPlan(log, m, group, reason) - } - } - } - } - } - } - return nil - }) - return newPlan, upgradeNotAllowed, fromVersion, toVersion, fromLicense, toLicense - } - - if newPlan, upgradeNotAllowed, fromVersion, toVersion, fromLicense, toLicense := createRotateOrUpgradePlan(); upgradeNotAllowed { - // Upgrade is needed, but not allowed - context.CreateEvent(k8sutil.NewUpgradeNotAllowedEvent(apiObject, fromVersion, toVersion, fromLicense, toLicense)) - } else if len(newPlan) > 0 { - if clusterReadyForUpgrade(context) { - // Use the new plan - plan = newPlan - } else { - log.Info().Msg("Pod needs upgrade but cluster is not ready. Either some shards are not in sync or some member is not ready.") - } - } + plan = createRotateOrUpgradePlan(log, apiObject, spec, status, context, pods) } // Check for the need to rotate TLS certificate of a members @@ -243,265 +158,6 @@ func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, return plan, true } -// clusterReadyForUpgrade returns true if the cluster is ready for the next update, that is: -// - all shards are in sync -// - all members are ready and fine -func clusterReadyForUpgrade(context PlanBuilderContext) bool { - status, _ := context.GetStatus() - allInSync := context.GetShardSyncStatus() - return allInSync && status.Conditions.IsTrue(api.ConditionTypeReady) -} - -// podNeedsUpgrading decides if an upgrade of the pod is needed (to comply with -// the given spec) and if that is allowed. -func podNeedsUpgrading(log zerolog.Logger, p v1.Pod, spec api.DeploymentSpec, images api.ImageInfoList) upgradeDecision { - if c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName); found { - specImageInfo, found := images.GetByImage(spec.GetImage()) - if !found { - return upgradeDecision{UpgradeNeeded: false} - } - podImageInfo, found := images.GetByImageID(c.Image) - if !found { - return upgradeDecision{UpgradeNeeded: false} - } - if specImageInfo.ImageID == podImageInfo.ImageID { - // No change - return upgradeDecision{UpgradeNeeded: false} - } - // Image changed, check if change is allowed - specVersion := specImageInfo.ArangoDBVersion - podVersion := podImageInfo.ArangoDBVersion - asLicense := func(info api.ImageInfo) upgraderules.License { - if info.Enterprise { - return upgraderules.LicenseEnterprise - } - return upgraderules.LicenseCommunity - } - specLicense := asLicense(specImageInfo) - podLicense := asLicense(podImageInfo) - if err := upgraderules.CheckUpgradeRulesWithLicense(podVersion, specVersion, podLicense, specLicense); err != nil { - // E.g. 3.x -> 4.x, we cannot allow automatically - return upgradeDecision{ - FromVersion: podVersion, - FromLicense: podLicense, - ToVersion: specVersion, - ToLicense: specLicense, - UpgradeNeeded: true, - UpgradeAllowed: false, - } - } - if specVersion.Major() != podVersion.Major() || specVersion.Minor() != podVersion.Minor() { - // Is allowed, with `--database.auto-upgrade` - log.Info().Str("spec-version", string(specVersion)).Str("pod-version", string(podVersion)). - Int("spec-version.major", specVersion.Major()).Int("spec-version.minor", specVersion.Minor()). - Int("pod-version.major", podVersion.Major()).Int("pod-version.minor", podVersion.Minor()). - Str("pod", p.GetName()).Msg("Deciding to do a upgrade with --auto-upgrade") - return upgradeDecision{ - FromVersion: podVersion, - FromLicense: podLicense, - ToVersion: specVersion, - ToLicense: specLicense, - UpgradeNeeded: true, - UpgradeAllowed: true, - AutoUpgradeNeeded: true, - } - } - // Patch version change, rotate only - return upgradeDecision{ - FromVersion: podVersion, - FromLicense: podLicense, - ToVersion: specVersion, - ToLicense: specLicense, - UpgradeNeeded: true, - UpgradeAllowed: true, - AutoUpgradeNeeded: false, - } - } - return upgradeDecision{UpgradeNeeded: false} -} - -// podNeedsRotation returns true when the specification of the -// given pod differs from what it should be according to the -// given deployment spec. -// When true is returned, a reason for the rotation is already returned. -func podNeedsRotation(log zerolog.Logger, p v1.Pod, apiObject metav1.Object, spec api.DeploymentSpec, - group api.ServerGroup, status api.DeploymentStatus, id string, - context PlanBuilderContext) (bool, string) { - groupSpec := spec.GetServerGroupSpec(group) - - // Check image pull policy - c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName) - if found { - if c.ImagePullPolicy != spec.GetImagePullPolicy() { - return true, "Image pull policy changed" - } - } else { - return true, "Server container not found" - } - - podImageInfo, found := status.Images.GetByImageID(c.Image) - if !found { - return false, "Server Image not found" - } - - if group.IsExportMetrics() { - e, hasExporter := k8sutil.GetContainerByName(&p, k8sutil.ExporterContainerName) - - if spec.Metrics.IsEnabled() { - if !hasExporter { - return true, "Exporter configuration changed" - } - - if spec.Metrics.HasImage() { - if e.Image != spec.Metrics.GetImage() { - return true, "Exporter image changed" - } - } - } else if hasExporter { - return true, "Exporter was disabled" - } - } - - // Check arguments - expectedArgs := strings.Join(context.GetExpectedPodArguments(apiObject, spec, group, status.Members.Agents, id, podImageInfo.ArangoDBVersion), " ") - actualArgs := strings.Join(getContainerArgs(c), " ") - if expectedArgs != actualArgs { - log.Debug(). - Str("actual-args", actualArgs). - Str("expected-args", expectedArgs). - Msg("Arguments changed. Rotation needed.") - return true, "Arguments changed" - } - - // Check service account - if normalizeServiceAccountName(p.Spec.ServiceAccountName) != normalizeServiceAccountName(groupSpec.GetServiceAccountName()) { - return true, "ServiceAccountName changed" - } - - // Check priorities - if groupSpec.PriorityClassName != p.Spec.PriorityClassName { - return true, "Pod priority changed" - } - - // Check resource requirements - var resources v1.ResourceRequirements - if groupSpec.HasVolumeClaimTemplate() { - resources = groupSpec.Resources // If there is a volume claim template compare all resources - } else { - resources = k8sutil.ExtractPodResourceRequirement(groupSpec.Resources) - } - - if resourcesRequireRotation(resources, k8sutil.GetArangoDBContainerFromPod(&p).Resources) { - return true, "Resource Requirements changed" - } - - var memberStatus, _, _ = status.Members.MemberStatusByPodName(p.GetName()) - if memberStatus.SideCarSpecs == nil { - memberStatus.SideCarSpecs = make(map[string]v1.Container) - } - - // Check for missing side cars in - for _, specSidecar := range groupSpec.GetSidecars() { - var stateSidecar v1.Container - if stateSidecar, found = memberStatus.SideCarSpecs[specSidecar.Name]; !found { - return true, "Sidecar " + specSidecar.Name + " not found in running pod " + p.GetName() - } - if sideCarRequireRotation(specSidecar.DeepCopy(), &stateSidecar) { - return true, "Sidecar " + specSidecar.Name + " requires rotation" - } - } - - for name := range memberStatus.SideCarSpecs { - var found = false - for _, specSidecar := range groupSpec.GetSidecars() { - if name == specSidecar.Name { - found = true - break - } - } - if !found { - return true, "Sidecar " + name + " no longer in specification" - } - } - - return false, "" -} - -// sideCarRequireRotation checks if side car requires rotation including default parameters -func sideCarRequireRotation(wanted, given *v1.Container) bool { - return !reflect.DeepEqual(wanted, given) -} - -// resourcesRequireRotation returns true if the resource requirements have changed such that a rotation is required -func resourcesRequireRotation(wanted, given v1.ResourceRequirements) bool { - checkList := func(wanted, given v1.ResourceList) bool { - for k, v := range wanted { - if gv, ok := given[k]; !ok { - return true - } else if v.Cmp(gv) != 0 { - return true - } - } - - return false - } - - return checkList(wanted.Limits, given.Limits) || checkList(wanted.Requests, given.Requests) -} - -// normalizeServiceAccountName replaces default with empty string, otherwise returns the input. -func normalizeServiceAccountName(name string) string { - if name == "default" { - return "" - } - return "" -} - -// createScalePlan creates a scaling plan for a single server group -func createScalePlan(log zerolog.Logger, members api.MemberStatusList, group api.ServerGroup, count int) api.Plan { - var plan api.Plan - if len(members) < count { - // Scale up - toAdd := count - len(members) - for i := 0; i < toAdd; i++ { - plan = append(plan, api.NewAction(api.ActionTypeAddMember, group, "")) - } - log.Debug(). - Int("count", count). - Int("actual-count", len(members)). - Int("delta", toAdd). - Str("role", group.AsRole()). - Msg("Creating scale-up plan") - } else if len(members) > count { - // Note, we scale down 1 member at a time - if m, err := members.SelectMemberToRemove(); err != nil { - log.Warn().Err(err).Str("role", group.AsRole()).Msg("Failed to select member to remove") - } else { - - log.Debug(). - Str("member-id", m.ID). - Str("phase", string(m.Phase)). - Msg("Found member to remove") - if group == api.ServerGroupDBServers { - plan = append(plan, - api.NewAction(api.ActionTypeCleanOutMember, group, m.ID), - ) - } - plan = append(plan, - api.NewAction(api.ActionTypeShutdownMember, group, m.ID), - api.NewAction(api.ActionTypeRemoveMember, group, m.ID), - ) - log.Debug(). - Int("count", count). - Int("actual-count", len(members)). - Str("role", group.AsRole()). - Str("member-id", m.ID). - Msg("Creating scale-down plan") - } - } - return plan -} - // createRotateMemberPlan creates a plan to rotate (stop-recreate-start) an existing // member. func createRotateMemberPlan(log zerolog.Logger, member api.MemberStatus, @@ -517,36 +173,3 @@ func createRotateMemberPlan(log zerolog.Logger, member api.MemberStatus, } return plan } - -// createUpgradeMemberPlan creates a plan to upgrade (stop-recreateWithAutoUpgrade-stop-start) an existing -// member. -func createUpgradeMemberPlan(log zerolog.Logger, member api.MemberStatus, - group api.ServerGroup, reason string, imageName string, status api.DeploymentStatus, rotateStatefull bool) api.Plan { - upgradeAction := api.ActionTypeUpgradeMember - if rotateStatefull || group.IsStateless() { - upgradeAction = api.ActionTypeRotateMember - } - log.Debug(). - Str("id", member.ID). - Str("role", group.AsRole()). - Str("reason", reason). - Str("action", string(upgradeAction)). - Msg("Creating upgrade plan") - plan := api.Plan{ - api.NewAction(upgradeAction, group, member.ID, reason), - api.NewAction(api.ActionTypeWaitForMemberUp, group, member.ID), - } - if status.CurrentImage == nil || status.CurrentImage.Image != imageName { - plan = append(api.Plan{ - api.NewAction(api.ActionTypeSetCurrentImage, group, "", reason).SetImage(imageName), - }, plan...) - } - return plan -} - -func getContainerArgs(c v1.Container) []string { - if len(c.Command) >= 1 { - return c.Command[1:] - } - return c.Args -} diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go new file mode 100644 index 000000000..9faf677de --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -0,0 +1,337 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + +package reconcile + +import ( + "reflect" + "strings" + + "github.com/arangodb/go-driver" + upgraderules "github.com/arangodb/go-upgrade-rules" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/rs/zerolog" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// createRotateOrUpgradePlan goes over all pods to check if an upgrade or rotate is needed. +func createRotateOrUpgradePlan(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, + status api.DeploymentStatus, context PlanBuilderContext, pods []v1.Pod) api.Plan { + + var newPlan api.Plan + var upgradeNotAllowed bool + var fromVersion, toVersion driver.Version + var fromLicense, toLicense upgraderules.License + + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + + for _, m := range members { + if m.Phase != api.MemberPhaseCreated || m.PodName == "" { + // Only rotate when phase is created + continue + } + + pod, found := k8sutil.GetPodByName(pods, m.PodName) + if !found { + continue + } + + // Got pod, compare it with what it should be + decision := podNeedsUpgrading(log, pod, spec, status.Images) + if decision.UpgradeNeeded && !decision.UpgradeAllowed { + // Oops, upgrade is not allowed + upgradeNotAllowed = true + fromVersion = decision.FromVersion + fromLicense = decision.FromLicense + toVersion = decision.ToVersion + toLicense = decision.ToLicense + return nil + } + + if len(newPlan) > 0 { + // Only rotate/upgrade 1 pod at a time + continue + } + + if decision.UpgradeNeeded { + // Yes, upgrade is needed (and allowed) + newPlan = createUpgradeMemberPlan(log, m, group, "Version upgrade", spec.GetImage(), status, + !decision.AutoUpgradeNeeded) + } else { + // Upgrade is not needed, see if rotation is needed + rotNeeded, reason := podNeedsRotation(log, pod, apiObject, spec, group, status, m.ID, context) + if rotNeeded { + newPlan = createRotateMemberPlan(log, m, group, reason) + } + } + } + return nil + }) + + if upgradeNotAllowed { + context.CreateEvent(k8sutil.NewUpgradeNotAllowedEvent(apiObject, fromVersion, toVersion, fromLicense, toLicense)) + } else if len(newPlan) > 0 { + if clusterReadyForUpgrade(context) { + // Use the new plan + return newPlan + } else { + log.Info().Msg("Pod needs upgrade but cluster is not ready. Either some shards are not in sync or some member is not ready.") + } + } + return nil +} + +// podNeedsUpgrading decides if an upgrade of the pod is needed (to comply with +// the given spec) and if that is allowed. +func podNeedsUpgrading(log zerolog.Logger, p v1.Pod, spec api.DeploymentSpec, images api.ImageInfoList) upgradeDecision { + if c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName); found { + specImageInfo, found := images.GetByImage(spec.GetImage()) + if !found { + return upgradeDecision{UpgradeNeeded: false} + } + podImageInfo, found := images.GetByImageID(c.Image) + if !found { + return upgradeDecision{UpgradeNeeded: false} + } + if specImageInfo.ImageID == podImageInfo.ImageID { + // No change + return upgradeDecision{UpgradeNeeded: false} + } + // Image changed, check if change is allowed + specVersion := specImageInfo.ArangoDBVersion + podVersion := podImageInfo.ArangoDBVersion + asLicense := func(info api.ImageInfo) upgraderules.License { + if info.Enterprise { + return upgraderules.LicenseEnterprise + } + return upgraderules.LicenseCommunity + } + specLicense := asLicense(specImageInfo) + podLicense := asLicense(podImageInfo) + if err := upgraderules.CheckUpgradeRulesWithLicense(podVersion, specVersion, podLicense, specLicense); err != nil { + // E.g. 3.x -> 4.x, we cannot allow automatically + return upgradeDecision{ + FromVersion: podVersion, + FromLicense: podLicense, + ToVersion: specVersion, + ToLicense: specLicense, + UpgradeNeeded: true, + UpgradeAllowed: false, + } + } + if specVersion.Major() != podVersion.Major() || specVersion.Minor() != podVersion.Minor() { + // Is allowed, with `--database.auto-upgrade` + log.Info().Str("spec-version", string(specVersion)).Str("pod-version", string(podVersion)). + Int("spec-version.major", specVersion.Major()).Int("spec-version.minor", specVersion.Minor()). + Int("pod-version.major", podVersion.Major()).Int("pod-version.minor", podVersion.Minor()). + Str("pod", p.GetName()).Msg("Deciding to do a upgrade with --auto-upgrade") + return upgradeDecision{ + FromVersion: podVersion, + FromLicense: podLicense, + ToVersion: specVersion, + ToLicense: specLicense, + UpgradeNeeded: true, + UpgradeAllowed: true, + AutoUpgradeNeeded: true, + } + } + // Patch version change, rotate only + return upgradeDecision{ + FromVersion: podVersion, + FromLicense: podLicense, + ToVersion: specVersion, + ToLicense: specLicense, + UpgradeNeeded: true, + UpgradeAllowed: true, + AutoUpgradeNeeded: false, + } + } + return upgradeDecision{UpgradeNeeded: false} +} + +// podNeedsRotation returns true when the specification of the +// given pod differs from what it should be according to the +// given deployment spec. +// When true is returned, a reason for the rotation is already returned. +func podNeedsRotation(log zerolog.Logger, p v1.Pod, apiObject metav1.Object, spec api.DeploymentSpec, + group api.ServerGroup, status api.DeploymentStatus, id string, + context PlanBuilderContext) (bool, string) { + groupSpec := spec.GetServerGroupSpec(group) + + // Check image pull policy + c, found := k8sutil.GetContainerByName(&p, k8sutil.ServerContainerName) + if found { + if c.ImagePullPolicy != spec.GetImagePullPolicy() { + return true, "Image pull policy changed" + } + } else { + return true, "Server container not found" + } + + podImageInfo, found := status.Images.GetByImageID(c.Image) + if !found { + return false, "Server Image not found" + } + + if group.IsExportMetrics() { + e, hasExporter := k8sutil.GetContainerByName(&p, k8sutil.ExporterContainerName) + + if spec.Metrics.IsEnabled() { + if !hasExporter { + return true, "Exporter configuration changed" + } + + if spec.Metrics.HasImage() { + if e.Image != spec.Metrics.GetImage() { + return true, "Exporter image changed" + } + } + + if k8sutil.IsResourceRequirementsChanged(spec.Metrics.Resources, e.Resources) { + return true, "Resources requirements have been changed for exporter" + } + } else if hasExporter { + return true, "Exporter was disabled" + } + } + + // Check arguments + expectedArgs := strings.Join(context.GetExpectedPodArguments(apiObject, spec, group, status.Members.Agents, id, podImageInfo.ArangoDBVersion), " ") + actualArgs := strings.Join(getContainerArgs(c), " ") + if expectedArgs != actualArgs { + log.Debug(). + Str("actual-args", actualArgs). + Str("expected-args", expectedArgs). + Msg("Arguments changed. Rotation needed.") + return true, "Arguments changed" + } + + // Check service account + if normalizeServiceAccountName(p.Spec.ServiceAccountName) != normalizeServiceAccountName(groupSpec.GetServiceAccountName()) { + return true, "ServiceAccountName changed" + } + + // Check priorities + if groupSpec.PriorityClassName != p.Spec.PriorityClassName { + return true, "Pod priority changed" + } + + // Check resource requirements + var resources v1.ResourceRequirements + if groupSpec.HasVolumeClaimTemplate() { + resources = groupSpec.Resources // If there is a volume claim template compare all resources + } else { + resources = k8sutil.ExtractPodResourceRequirement(groupSpec.Resources) + } + + if k8sutil.IsResourceRequirementsChanged(resources, k8sutil.GetArangoDBContainerFromPod(&p).Resources) { + return true, "Resource Requirements changed" + } + + var memberStatus, _, _ = status.Members.MemberStatusByPodName(p.GetName()) + if memberStatus.SideCarSpecs == nil { + memberStatus.SideCarSpecs = make(map[string]v1.Container) + } + + // Check for missing side cars in + for _, specSidecar := range groupSpec.GetSidecars() { + var stateSidecar v1.Container + if stateSidecar, found = memberStatus.SideCarSpecs[specSidecar.Name]; !found { + return true, "Sidecar " + specSidecar.Name + " not found in running pod " + p.GetName() + } + if sideCarRequireRotation(specSidecar.DeepCopy(), &stateSidecar) { + return true, "Sidecar " + specSidecar.Name + " requires rotation" + } + } + + for name := range memberStatus.SideCarSpecs { + var found = false + for _, specSidecar := range groupSpec.GetSidecars() { + if name == specSidecar.Name { + found = true + break + } + } + if !found { + return true, "Sidecar " + name + " no longer in specification" + } + } + + return false, "" +} + +// clusterReadyForUpgrade returns true if the cluster is ready for the next update, that is: +// - all shards are in sync +// - all members are ready and fine +func clusterReadyForUpgrade(context PlanBuilderContext) bool { + status, _ := context.GetStatus() + allInSync := context.GetShardSyncStatus() + return allInSync && status.Conditions.IsTrue(api.ConditionTypeReady) +} + +// sideCarRequireRotation checks if side car requires rotation including default parameters +func sideCarRequireRotation(wanted, given *v1.Container) bool { + return !reflect.DeepEqual(wanted, given) +} + +// normalizeServiceAccountName replaces default with empty string, otherwise returns the input. +func normalizeServiceAccountName(name string) string { + if name == "default" { + return "" + } + return "" +} + +// createUpgradeMemberPlan creates a plan to upgrade (stop-recreateWithAutoUpgrade-stop-start) an existing +// member. +func createUpgradeMemberPlan(log zerolog.Logger, member api.MemberStatus, + group api.ServerGroup, reason string, imageName string, status api.DeploymentStatus, rotateStatefull bool) api.Plan { + upgradeAction := api.ActionTypeUpgradeMember + if rotateStatefull || group.IsStateless() { + upgradeAction = api.ActionTypeRotateMember + } + log.Debug(). + Str("id", member.ID). + Str("role", group.AsRole()). + Str("reason", reason). + Str("action", string(upgradeAction)). + Msg("Creating upgrade plan") + plan := api.Plan{ + api.NewAction(upgradeAction, group, member.ID, reason), + api.NewAction(api.ActionTypeWaitForMemberUp, group, member.ID), + } + if status.CurrentImage == nil || status.CurrentImage.Image != imageName { + plan = append(api.Plan{ + api.NewAction(api.ActionTypeSetCurrentImage, group, "", reason).SetImage(imageName), + }, plan...) + } + return plan +} + +func getContainerArgs(c v1.Container) []string { + if len(c.Command) >= 1 { + return c.Command[1:] + } + return c.Args +} diff --git a/pkg/deployment/reconcile/plan_builder_scale.go b/pkg/deployment/reconcile/plan_builder_scale.go new file mode 100644 index 000000000..80e22e6e1 --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_scale.go @@ -0,0 +1,97 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + +package reconcile + +import ( + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func createScaleMemeberPlan(log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus) api.Plan { + + var plan api.Plan + + switch spec.GetMode() { + case api.DeploymentModeSingle: + // Never scale down + case api.DeploymentModeActiveFailover: + // Only scale singles + plan = append(plan, createScalePlan(log, status.Members.Single, api.ServerGroupSingle, spec.Single.GetCount())...) + case api.DeploymentModeCluster: + // Scale dbservers, coordinators + plan = append(plan, createScalePlan(log, status.Members.DBServers, api.ServerGroupDBServers, spec.DBServers.GetCount())...) + plan = append(plan, createScalePlan(log, status.Members.Coordinators, api.ServerGroupCoordinators, spec.Coordinators.GetCount())...) + } + if spec.GetMode().SupportsSync() { + // Scale syncmasters & syncworkers + plan = append(plan, createScalePlan(log, status.Members.SyncMasters, api.ServerGroupSyncMasters, spec.SyncMasters.GetCount())...) + plan = append(plan, createScalePlan(log, status.Members.SyncWorkers, api.ServerGroupSyncWorkers, spec.SyncWorkers.GetCount())...) + } + + return plan +} + +// createScalePlan creates a scaling plan for a single server group +func createScalePlan(log zerolog.Logger, members api.MemberStatusList, group api.ServerGroup, count int) api.Plan { + var plan api.Plan + if len(members) < count { + // Scale up + toAdd := count - len(members) + for i := 0; i < toAdd; i++ { + plan = append(plan, api.NewAction(api.ActionTypeAddMember, group, "")) + } + log.Debug(). + Int("count", count). + Int("actual-count", len(members)). + Int("delta", toAdd). + Str("role", group.AsRole()). + Msg("Creating scale-up plan") + } else if len(members) > count { + // Note, we scale down 1 member at a time + if m, err := members.SelectMemberToRemove(); err != nil { + log.Warn().Err(err).Str("role", group.AsRole()).Msg("Failed to select member to remove") + } else { + + log.Debug(). + Str("member-id", m.ID). + Str("phase", string(m.Phase)). + Msg("Found member to remove") + if group == api.ServerGroupDBServers { + plan = append(plan, + api.NewAction(api.ActionTypeCleanOutMember, group, m.ID), + ) + } + plan = append(plan, + api.NewAction(api.ActionTypeShutdownMember, group, m.ID), + api.NewAction(api.ActionTypeRemoveMember, group, m.ID), + ) + log.Debug(). + Int("count", count). + Int("actual-count", len(members)). + Str("role", group.AsRole()). + Str("member-id", m.ID). + Msg("Creating scale-down plan") + } + } + return plan +} diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index 330bf5b58..88c757529 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -29,7 +29,7 @@ import ( "github.com/arangodb/go-driver/agency" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" ) @@ -40,7 +40,7 @@ type ServerGroupIterator interface { // If the callback returns an error, this error is returned and no other server // groups are processed. // Groups are processed in this order: agents, single, dbservers, coordinators, syncmasters, syncworkers - ForeachServerGroup(cb func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error, status *api.DeploymentStatus) error + ForeachServerGroup(cb api.ServerGroupFunc, status *api.DeploymentStatus) error } // Context provides all functions needed by the Resources service diff --git a/pkg/deployment/resources/exporter.go b/pkg/deployment/resources/exporter.go new file mode 100644 index 000000000..eb44db660 --- /dev/null +++ b/pkg/deployment/resources/exporter.go @@ -0,0 +1,96 @@ +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + +package resources + +import ( + "path/filepath" + "sort" + "strconv" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" +) + +// ArangodbExporterContainer creates metrics container +func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sutil.HTTPProbeConfig, + resources v1.ResourceRequirements) v1.Container { + + c := v1.Container{ + Name: k8sutil.ExporterContainerName, + Image: image, + Command: append([]string{"/app/arangodb-exporter"}, args...), + Ports: []v1.ContainerPort{ + { + Name: "exporter", + ContainerPort: int32(k8sutil.ArangoExporterPort), + Protocol: v1.ProtocolTCP, + }, + }, + Resources: k8sutil.ExtractPodResourceRequirement(resources), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: k8sutil.SecurityContextWithoutCapabilities(), + } + + if livenessProbe != nil { + c.LivenessProbe = livenessProbe.Create() + } + + return c +} + +func createExporterArgs(isSecure bool) []string { + tokenpath := filepath.Join(k8sutil.ExporterJWTVolumeMountDir, constants.SecretKeyToken) + options := make([]optionPair, 0, 64) + scheme := "http" + if isSecure { + scheme = "https" + } + options = append(options, + optionPair{"--arangodb.jwt-file", tokenpath}, + optionPair{"--arangodb.endpoint", scheme + "://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, + ) + keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) + if isSecure { + options = append(options, + optionPair{"--ssl.keyfile", keyPath}, + ) + } + args := make([]string, 0, 2+len(options)) + sort.Slice(options, func(i, j int) bool { + return options[i].CompareTo(options[j]) < 0 + }) + for _, o := range options { + args = append(args, o.Key+"="+o.Value) + } + + return args +} + +func createExporterLivenessProbe(isSecure bool) *k8sutil.HTTPProbeConfig { + probeCfg := &k8sutil.HTTPProbeConfig{ + LocalPath: "/", + Port: k8sutil.ArangoExporterPort, + Secure: isSecure, + } + + return probeCfg +} diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index e24650bbb..ab3d12c9f 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -331,34 +331,6 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou return args } -func createExporterArgs(isSecure bool) []string { - tokenpath := filepath.Join(k8sutil.ExporterJWTVolumeMountDir, constants.SecretKeyToken) - options := make([]optionPair, 0, 64) - scheme := "http" - if isSecure { - scheme = "https" - } - options = append(options, - optionPair{"--arangodb.jwt-file", tokenpath}, - optionPair{"--arangodb.endpoint", scheme + "://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, - ) - keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) - if isSecure { - options = append(options, - optionPair{"--ssl.keyfile", keyPath}, - ) - } - args := make([]string, 0, 2+len(options)) - sort.Slice(options, func(i, j int) bool { - return options[i].CompareTo(options[j]) < 0 - }) - for _, o := range options { - args = append(args, o.Key+"="+o.Value) - } - - return args -} - // createLivenessProbe creates configuration for a liveness probe of a server in the given group. func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.ServerGroup) (*k8sutil.HTTPProbeConfig, error) { groupspec := spec.GetServerGroupSpec(group) @@ -524,16 +496,6 @@ func (r *Resources) CreatePodTolerations(group api.ServerGroup, groupSpec api.Se return tolerations } -func createExporterLivenessProbe(isSecure bool) *k8sutil.HTTPProbeConfig { - probeCfg := &k8sutil.HTTPProbeConfig{ - LocalPath: "/", - Port: k8sutil.ArangoExporterPort, - Secure: isSecure, - } - - return probeCfg -} - // createPodForMember creates all Pods listed in member status func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once) error { kubecli := r.context.GetKubeCli() @@ -615,29 +577,11 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, } } - var exporter *k8sutil.ArangodbExporterContainerConf - - if spec.Metrics.IsEnabled() { - if group.IsExportMetrics() { - image := spec.GetImage() - if spec.Metrics.HasImage() { - image = spec.Metrics.GetImage() - } - exporter = &k8sutil.ArangodbExporterContainerConf{ - Args: createExporterArgs(spec.IsSecure()), - JWTTokenSecretName: spec.Metrics.GetJWTTokenSecretName(), - LivenessProbe: createExporterLivenessProbe(spec.IsSecure()), - Image: image, - } - } - } - memberPod := MemberArangoDPod{ status: m, tlsKeyfileSecretName: tlsKeyfileSecretName, rocksdbEncryptionSecretName: rocksdbEncryptionSecretName, clusterJWTSecretName: clusterJWTSecretName, - exporter: exporter, groupSpec: groupSpec, spec: spec, group: group, diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index e81f3aedf..d9f8b2039 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -41,7 +41,6 @@ type MemberArangoDPod struct { tlsKeyfileSecretName string rocksdbEncryptionSecretName string clusterJWTSecretName string - exporter *k8sutil.ArangodbExporterContainerConf groupSpec api.ServerGroupSpec spec api.DeploymentSpec group api.ServerGroup @@ -162,11 +161,17 @@ func (m *MemberArangoDPod) GetServiceAccountName() string { } func (m *MemberArangoDPod) GetSidecars(pod *v1.Pod) { - if m.exporter != nil { - // Metrics sidecar - c := k8sutil.ArangodbexporterContainer(m.exporter.Image, m.exporter.Args, m.exporter.Env, m.exporter.LivenessProbe) - if m.exporter.JWTTokenSecretName != "" { + if isMetricsEnabledForGroup(m.spec, m.group) { + image := m.spec.GetImage() + if m.spec.Metrics.HasImage() { + image = m.spec.Metrics.GetImage() + } + + c := ArangodbExporterContainer(image, createExporterArgs(m.spec.IsSecure()), + createExporterLivenessProbe(m.spec.IsSecure()), m.spec.Metrics.Resources) + + if m.spec.Metrics.GetJWTTokenSecretName() != "" { c.VolumeMounts = append(c.VolumeMounts, k8sutil.ExporterJWTVolumeMount()) } @@ -218,9 +223,12 @@ func (m *MemberArangoDPod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { volumeMounts = append(volumeMounts, k8sutil.RocksdbEncryptionVolumeMount()) } - if m.exporter != nil && m.exporter.JWTTokenSecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, m.exporter.JWTTokenSecretName) - volumes = append(volumes, vol) + if isMetricsEnabledForGroup(m.spec, m.group) { + token := m.spec.Metrics.GetJWTTokenSecretName() + if token != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token) + volumes = append(volumes, vol) + } } if m.clusterJWTSecretName != "" { @@ -245,7 +253,7 @@ func (m *MemberArangoDPod) GetInitContainers() ([]v1.Container, error) { lifecycleImage := m.resources.context.GetLifecycleImage() if lifecycleImage != "" { - c, err := k8sutil.InitLifecycleContainer(lifecycleImage) + c, err := k8sutil.InitLifecycleContainer(lifecycleImage, &m.spec.Lifecycle.Resources) if err != nil { return nil, err } @@ -281,3 +289,7 @@ func (m *MemberArangoDPod) GetContainerCreator() k8sutil.ContainerCreator { groupSpec: m.groupSpec, } } + +func isMetricsEnabledForGroup(spec api.DeploymentSpec, group api.ServerGroup) bool { + return spec.Metrics.IsEnabled() && group.IsExportMetrics() +} diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 8dea1fd7e..3bea6c43d 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -195,7 +195,7 @@ func (m *MemberSyncPod) GetInitContainers() ([]v1.Container, error) { lifecycleImage := m.resources.context.GetLifecycleImage() if lifecycleImage != "" { - c, err := k8sutil.InitLifecycleContainer(lifecycleImage) + c, err := k8sutil.InitLifecycleContainer(lifecycleImage, &m.spec.Lifecycle.Resources) if err != nil { return nil, err } diff --git a/pkg/util/k8sutil/container.go b/pkg/util/k8sutil/container.go index b018eff97..ee33d930a 100644 --- a/pkg/util/k8sutil/container.go +++ b/pkg/util/k8sutil/container.go @@ -22,9 +22,7 @@ package k8sutil -import ( - "k8s.io/api/core/v1" -) +import v1 "k8s.io/api/core/v1" // GetContainerByName returns the container in the given pod with the given name. // Returns false if not found. @@ -36,3 +34,20 @@ func GetContainerByName(p *v1.Pod, name string) (v1.Container, bool) { } return v1.Container{}, false } + +// IsResourceRequirementsChanged returns true if the resource requirements have changed. +func IsResourceRequirementsChanged(wanted, given v1.ResourceRequirements) bool { + checkList := func(wanted, given v1.ResourceList) bool { + for k, v := range wanted { + if gv, ok := given[k]; !ok { + return true + } else if v.Cmp(gv) != 0 { + return true + } + } + + return false + } + + return checkList(wanted.Limits, given.Limits) || checkList(wanted.Requests, given.Requests) +} diff --git a/pkg/util/k8sutil/lifecycle.go b/pkg/util/k8sutil/lifecycle.go index 1e86a8ebb..2de9ec452 100644 --- a/pkg/util/k8sutil/lifecycle.go +++ b/pkg/util/k8sutil/lifecycle.go @@ -38,21 +38,25 @@ const ( ) // InitLifecycleContainer creates an init-container to copy the lifecycle binary to a shared volume. -func InitLifecycleContainer(image string) (v1.Container, error) { +func InitLifecycleContainer(image string, resources *v1.ResourceRequirements) (v1.Container, error) { binaryPath, err := os.Executable() if err != nil { return v1.Container{}, maskAny(err) } c := v1.Container{ - Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", lifecycleVolumeMountDir), - Name: initLifecycleContainerName, - Image: image, - ImagePullPolicy: v1.PullIfNotPresent, + Name: initLifecycleContainerName, + Image: image, + Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", lifecycleVolumeMountDir), VolumeMounts: []v1.VolumeMount{ LifecycleVolumeMount(), }, + ImagePullPolicy: v1.PullIfNotPresent, SecurityContext: SecurityContextWithoutCapabilities(), } + + if resources != nil { + c.Resources = ExtractPodResourceRequirement(*resources) + } return c, nil } diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index f75cd52c2..438d82718 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -28,6 +28,8 @@ import ( "strings" "time" + "k8s.io/apimachinery/pkg/api/resource" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -52,13 +54,6 @@ const ( MasterJWTSecretVolumeMountDir = "/secrets/master/jwt" ) -// EnvValue is a helper structure for environment variable sources. -type EnvValue struct { - Value string // If set, the environment value gets this value - SecretName string // If set, the environment value gets its value from a secret with this name - SecretKey string // Key inside secret to fill into the envvar. Only relevant is SecretName is set. -} - type PodCreator interface { Init(*v1.Pod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) @@ -84,27 +79,6 @@ type ContainerCreator interface { GetEnvs() []v1.EnvVar } -// CreateEnvVar creates an EnvVar structure for given key from given EnvValue. -func (v EnvValue) CreateEnvVar(key string) v1.EnvVar { - ev := v1.EnvVar{ - Name: key, - } - if ev.Value != "" { - ev.Value = v.Value - } else if v.SecretName != "" { - //return CreateEnvSecretKeySelector(key, v.SecretName, v.SecretKey) - ev.ValueFrom = &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ - Name: v.SecretName, - }, - Key: v.SecretKey, - }, - } - } - return ev -} - // IsPodReady returns true if the PodReady condition on // the given pod is set to true. func IsPodReady(pod *v1.Pod) bool { @@ -112,6 +86,17 @@ func IsPodReady(pod *v1.Pod) bool { return condition != nil && condition.Status == v1.ConditionTrue } +// GetPodByName returns pod if it exists among the pods' list +// Returns false if not found. +func GetPodByName(pods []v1.Pod, podName string) (v1.Pod, bool) { + for _, pod := range pods { + if pod.GetName() == podName { + return pod, true + } + } + return v1.Pod{}, false +} + // IsPodSucceeded returns true if the arangodb container of the pod // has terminated with exit code 0. func IsPodSucceeded(pod *v1.Pod) bool { @@ -291,13 +276,23 @@ func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool command = fmt.Sprintf("test -f %s || echo '%s' > %s", uuidFile, id, uuidFile) } c := v1.Container{ + Name: name, + Image: alpineImage, Command: []string{ "/bin/sh", "-c", command, }, - Name: name, - Image: alpineImage, + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("10Mi"), + }, + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("50Mi"), + }, + }, VolumeMounts: []v1.VolumeMount{ ArangodVolumeMount(), }, @@ -360,30 +355,6 @@ func NewContainer(args []string, containerCreator ContainerCreator) (v1.Containe }, nil } -func ArangodbexporterContainer(image string, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { - c := v1.Container{ - Command: append([]string{"/app/arangodb-exporter"}, args...), - Name: ExporterContainerName, - Image: image, - ImagePullPolicy: v1.PullIfNotPresent, - Ports: []v1.ContainerPort{ - { - Name: "exporter", - ContainerPort: int32(ArangoExporterPort), - Protocol: v1.ProtocolTCP, - }, - }, - SecurityContext: SecurityContextWithoutCapabilities(), - } - for k, v := range env { - c.Env = append(c.Env, v.CreateEnvVar(k)) - } - if livenessProbe != nil { - c.LivenessProbe = livenessProbe.Create() - } - return c -} - // NewPod creates a basic Pod for given settings. func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) v1.Pod { @@ -419,15 +390,6 @@ func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) v1. return p } -// ArangodbExporterContainerConf contains configuration of the exporter container -type ArangodbExporterContainerConf struct { - Args []string - Env map[string]EnvValue - JWTTokenSecretName string - LivenessProbe *HTTPProbeConfig - Image string -} - // CreatePod adds an owner to the given pod and calls the k8s api-server to created it. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. diff --git a/tests/metrics_test.go b/tests/metrics_test.go index 370fb9a62..d82f624f7 100644 --- a/tests/metrics_test.go +++ b/tests/metrics_test.go @@ -27,12 +27,20 @@ import ( "testing" "time" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/require" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" kubeArangoClient "github.com/arangodb/kube-arangodb/pkg/client" "github.com/arangodb/kube-arangodb/pkg/util" "github.com/dchest/uniuri" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" ) func TestAddingMetrics(t *testing.T) { @@ -50,53 +58,82 @@ func TestAddingMetrics(t *testing.T) { // Create deployment deployment, err := c.DatabaseV1().ArangoDeployments(ns).Create(depl) - if err != nil { - t.Fatalf("Create deployment failed: %v", err) - } + require.NoErrorf(t, err, "Create deployment failed") defer deferedCleanupDeployment(c, depl.GetName(), ns) // Wait for deployment to be ready deployment, err = waitUntilDeployment(c, depl.GetName(), ns, deploymentIsReady()) - if err != nil { - t.Fatalf("Deployment not running in time: %v", err) - } + require.NoErrorf(t, err, "Deployment not running in time") // Create a database client ctx := context.Background() DBClient := mustNewArangodDatabaseClient(ctx, kubecli, deployment, t, nil) - if err := waitUntilArangoDeploymentHealthy(deployment, DBClient, kubecli, ""); err != nil { - t.Fatalf("Deployment not healthy in time: %v", err) - } + err = waitUntilArangoDeploymentHealthy(deployment, DBClient, kubecli, "") + require.NoErrorf(t, err, "Deployment not healthy in time") // Try to switch on metrics: - deployment, err = updateDeployment(c, depl.GetName(), ns, - func(depl *api.DeploymentSpec) { - depl.Metrics = api.MetricsSpec{ - Enabled: util.NewBool(true), - Image: util.NewString("arangodb/arangodb-exporter:0.1.6"), - } - }) - if err != nil { - t.Fatalf("Failed to add metrics") - } else { - t.Log("Updated deployment by adding metrics") + expectedResourceRequirement := corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("100m"), + }, } - if err := waitUntilArangoDeploymentHealthy(deployment, DBClient, kubecli, ""); err != nil { - t.Errorf("Deployment not healthy in time: %v", err) - } else { - t.Log("Deployment healthy") + deployment, err = updateDeployment(c, depl.GetName(), ns, func(depl *api.DeploymentSpec) { + depl.Metrics = api.MetricsSpec{ + Enabled: util.NewBool(true), + Image: util.NewString("arangodb/arangodb-exporter:0.1.6"), + Resources: expectedResourceRequirement, + } + }) + require.NoErrorf(t, err, "Failed to add metrics") + t.Log("Updated deployment by adding metrics") + + var resourcesRequirementsExporterCheck api.ServerGroupFunc = func(group api.ServerGroup, spec api.ServerGroupSpec, + status *api.MemberStatusList) error { + + if !group.IsExportMetrics() { + return nil + } + for _, m := range *status { + pod, err := kubecli.CoreV1().Pods(ns).Get(m.PodName, metav1.GetOptions{}) + if err != nil { + return err + } + exporter, found := k8sutil.GetContainerByName(pod, k8sutil.ExporterContainerName) + if !found { + return fmt.Errorf("expected exporter to be enabled") + } + + if k8sutil.IsResourceRequirementsChanged(expectedResourceRequirement, exporter.Resources) { + return fmt.Errorf("resources have not been changed: expected %v, actual %v", + expectedResourceRequirement, exporter.Resources) + } + } + return nil } + _, err = waitUntilDeploymentMembers(c, deployment.GetName(), ns, resourcesRequirementsExporterCheck, 7*time.Minute) + require.NoError(t, err) + + expectedResourceRequirement.Requests[v1.ResourceCPU] = resource.MustParse("110m") + deployment, err = updateDeployment(c, depl.GetName(), ns, func(depl *api.DeploymentSpec) { + depl.Metrics.Resources = expectedResourceRequirement + }) + require.NoErrorf(t, err, "failed to change resource requirements for metrics") + t.Log("Updated deployment by changing metrics") + _, err = waitUntilDeploymentMembers(c, deployment.GetName(), ns, resourcesRequirementsExporterCheck, 7*time.Minute) + require.NoError(t, err) + + err = waitUntilArangoDeploymentHealthy(deployment, DBClient, kubecli, "") + require.NoErrorf(t, err, "Deployment not healthy in time") + t.Log("Deployment healthy") + _, err = waitUntilService(kubecli, depl.GetName()+"-exporter", ns, func(service *corev1.Service) error { return nil }, time.Second*30) - if err != nil { - t.Errorf("Exporter service did not show up in time") - } else { - t.Log("Found exporter service") - } + require.NoErrorf(t, err, "Exporter service did not show up in time") + t.Log("Found exporter service") _, err = waitUntilEndpoints(kubecli, depl.GetName()+"-exporter", ns, func(endpoints *corev1.Endpoints) error { @@ -110,11 +147,8 @@ func TestAddingMetrics(t *testing.T) { } return nil }, time.Second*360) // needs a full rotation with extra containers - if err != nil { - t.Errorf("Exporter endpoints did not show up in time") - } else { - t.Log("Found exporter endpoints") - } + require.NoErrorf(t, err, "Exporter endpoints did not show up in time") + t.Log("Found exporter endpoints") // Cleanup removeDeployment(c, depl.GetName(), ns) diff --git a/tests/predicates.go b/tests/predicates.go index 3b7f06fb4..0393f9862 100644 --- a/tests/predicates.go +++ b/tests/predicates.go @@ -25,7 +25,6 @@ package tests import ( "fmt" - v1 "k8s.io/api/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -46,22 +45,6 @@ func deploymentIsReady() func(*api.ArangoDeployment) error { } } -func resourcesRequireRotation(wanted, given v1.ResourceRequirements) bool { - checkList := func(wanted, given v1.ResourceList) bool { - for k, v := range wanted { - if gv, ok := given[k]; !ok { - return true - } else if v.Cmp(gv) != 0 { - return true - } - } - - return false - } - - return checkList(wanted.Limits, given.Limits) || checkList(wanted.Requests, given.Requests) -} - func resourcesAsRequested(kubecli kubernetes.Interface, ns string) func(obj *api.ArangoDeployment) error { return func(obj *api.ArangoDeployment) error { return obj.ForeachServerGroup(func(group api.ServerGroup, spec api.ServerGroupSpec, status *api.MemberStatusList) error { @@ -77,7 +60,7 @@ func resourcesAsRequested(kubecli kubernetes.Interface, ns string) func(obj *api return fmt.Errorf("Container not found: %s", m.PodName) } - if resourcesRequireRotation(spec.Resources, c.Resources) { + if k8sutil.IsResourceRequirementsChanged(spec.Resources, c.Resources) { return fmt.Errorf("Container of Pod %s need rotation", m.PodName) } } diff --git a/tests/test_util.go b/tests/test_util.go index cdb4eb252..523bff4a6 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -26,8 +26,6 @@ import ( "context" "crypto/tls" "fmt" - "github.com/arangodb/kube-arangodb/pkg/apis/deployment" - "github.com/arangodb/kube-arangodb/pkg/apis/replication" "net" "os" "reflect" @@ -37,6 +35,9 @@ import ( "testing" "time" + "github.com/arangodb/kube-arangodb/pkg/apis/deployment" + "github.com/arangodb/kube-arangodb/pkg/apis/replication" + v1 "k8s.io/api/core/v1" "k8s.io/client-go/kubernetes" @@ -294,6 +295,15 @@ func newDeployment(name string) *api.ArangoDeployment { return depl } +// waitUntilDeployment waits until a deployment members are ready with given name in given namespace +func waitUntilDeploymentMembers(cli versioned.Interface, deploymentName, ns string, cb api.ServerGroupFunc, + timeout ...time.Duration) (*api.ArangoDeployment, error) { + + return waitUntilDeployment(cli, deploymentName, ns, func(d *api.ArangoDeployment) error { + return d.ForeachServerGroup(cb, &d.Status) + }, timeout...) +} + // waitUntilDeployment waits until a deployment with given name in given namespace // reached a state where the given predicate returns true. func waitUntilDeployment(cli versioned.Interface, deploymentName, ns string, predicate func(*api.ArangoDeployment) error, timeout ...time.Duration) (*api.ArangoDeployment, error) {