From d1746b649ed43d429e490806fb4a8f3699a72b66 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Fri, 20 Mar 2020 13:37:48 +0000 Subject: [PATCH 1/6] Add AntiAffinity to GroupSpec --- pkg/apis/deployment/v1/server_group_spec.go | 2 + pkg/deployment/deployment_affinity_test.go | 319 ++++++++++++++++++ pkg/deployment/deployment_run_test.go | 26 +- pkg/deployment/deployment_suite_test.go | 1 + pkg/deployment/images.go | 71 ++-- pkg/deployment/pod/affinity.go | 131 +++++++ pkg/deployment/resources/pod_creator.go | 9 +- .../resources/pod_creator_arangod.go | 34 +- pkg/deployment/resources/pod_creator_sync.go | 76 +++-- pkg/util/k8sutil/pods.go | 204 +++++------ 10 files changed, 725 insertions(+), 148 deletions(-) create mode 100644 pkg/deployment/deployment_affinity_test.go create mode 100644 pkg/deployment/pod/affinity.go diff --git a/pkg/apis/deployment/v1/server_group_spec.go b/pkg/apis/deployment/v1/server_group_spec.go index 2875d5b49..45fd1368d 100644 --- a/pkg/apis/deployment/v1/server_group_spec.go +++ b/pkg/apis/deployment/v1/server_group_spec.go @@ -70,6 +70,8 @@ type ServerGroupSpec struct { VolumeClaimTemplate *v1.PersistentVolumeClaim `json:"volumeClaimTemplate,omitempty"` // VolumeResizeMode specified resize mode for pvc VolumeResizeMode *PVCResizeMode `json:"pvcResizeMode,omitempty"` + // AntiAffinity specified additional antiAffinity settings in ArangoDB Pod definitions + AntiAffinity *v1.PodAntiAffinity `json:"antiAffinity,omitempty"` // Sidecars specifies a list of additional containers to be started Sidecars []v1.Container `json:"sidecars,omitempty"` // SecurityContext specifies security context for group diff --git a/pkg/deployment/deployment_affinity_test.go b/pkg/deployment/deployment_affinity_test.go new file mode 100644 index 000000000..18cfc9b6e --- /dev/null +++ b/pkg/deployment/deployment_affinity_test.go @@ -0,0 +1,319 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package deployment + +import ( + "testing" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" +) + +func modifyAffinity(name, group string, required bool, role string, mods ... func(a *core.Affinity)) *core.Affinity { + affinity := k8sutil.CreateAffinity(name, group, + required, role) + + for _, mod := range mods { + mod(affinity) + } + + return affinity +} + +func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { + testAffinity := core.PodAffinityTerm{ + TopologyKey: "myTopologyKey", + } + + weight := core.WeightedPodAffinityTerm{ + Weight: 6, + PodAffinityTerm: testAffinity, + } + + testCases := []testCaseStruct{ + { + Name: "DBserver POD with antiAffinity required", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + AntiAffinity:&core.PodAntiAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ + testAffinity, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, testAffinity) + }), + }, + }, + }, + { + Name: "DBserver POD with antiAffinity prefered", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + AntiAffinity:&core.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + weight, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, weight) + }), + }, + }, + }, + { + Name: "DBserver POD with antiAffinity both", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + AntiAffinity:&core.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + weight, + }, + RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ + testAffinity, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, weight) + a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, testAffinity) + }), + }, + }, + }, + { + Name: "DBserver POD with antiAffinity mixed", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + AntiAffinity:&core.PodAntiAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + weight, + weight, + weight, + weight, + }, + RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ + testAffinity, + testAffinity, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution, weight, weight, weight, weight) + a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, testAffinity, testAffinity) + }), + }, + }, + }, + } + + runTestCases(t, testCases...) +} diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index 5797dc5f9..1a9db65e4 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -23,7 +23,11 @@ package deployment import ( + "encoding/json" "fmt" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" "testing" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -70,7 +74,10 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { pods, err := d.deps.KubeCli.CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) require.NoError(t, err) require.Len(t, pods.Items, 1) - require.Equal(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) + if util.BoolOrDefault(testCase.CompareChecksum, true) { + compareSpec(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) + } + require.Equal(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) require.Equal(t, testCase.ExpectedPod.ObjectMeta, pods.Items[0].ObjectMeta) if len(testCase.ExpectedEvent) > 0 { @@ -106,3 +113,20 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { } }) } + +func compareSpec(t *testing.T, a, b core.PodSpec) { + ac, err := k8sutil.GetPodSpecChecksum(a) + require.NoError(t, err) + + bc, err := k8sutil.GetPodSpecChecksum(b) + require.NoError(t, err) + + aj, err := json.Marshal(a) + require.NoError(t, err) + + bj, err := json.Marshal(b) + require.NoError(t, err) + + require.Equal(t, string(aj), string(bj)) + require.Equal(t, ac, bc) +} \ No newline at end of file diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index f9ba9fe39..f5e6790dd 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -65,6 +65,7 @@ type testCaseStruct struct { ArangoDeployment *api.ArangoDeployment Helper func(*testing.T, *Deployment, *testCaseStruct) config Config + CompareChecksum *bool ExpectedError error ExpectedEvent string ExpectedPod core.Pod diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 8856c4df1..4fd7a900e 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -26,11 +26,12 @@ import ( "context" "crypto/sha1" "fmt" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "strings" "time" "github.com/rs/zerolog" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" @@ -41,8 +42,11 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) +var _ k8sutil.PodCreator = &ImageUpdatePod{} + type ImageUpdatePod struct { spec api.DeploymentSpec + apiObject k8sutil.APIObject image string } @@ -193,6 +197,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima imagePod := ImageUpdatePod{ spec: ib.Spec, image: image, + apiObject: ib.APIObject, } pod, err := resources.RenderArangoPod(ib.APIObject, role, id, podName, args, &imagePod) @@ -213,14 +218,14 @@ func (a *ArangoDImageUpdateContainer) GetExecutor() string { return resources.ArangoDExecutor } -func (a *ArangoDImageUpdateContainer) GetProbes() (*v1.Probe, *v1.Probe, error) { +func (a *ArangoDImageUpdateContainer) GetProbes() (*core.Probe, *core.Probe, error) { return nil, nil, nil } -func (a *ArangoDImageUpdateContainer) GetResourceRequirements() v1.ResourceRequirements { - return v1.ResourceRequirements{ - Limits: make(v1.ResourceList), - Requests: make(v1.ResourceList), +func (a *ArangoDImageUpdateContainer) GetResourceRequirements() core.ResourceRequirements { + return core.ResourceRequirements{ + Limits: make(core.ResourceList), + Requests: make(core.ResourceList), } } @@ -228,8 +233,8 @@ func (a *ArangoDImageUpdateContainer) GetImage() string { return a.image } -func (a *ArangoDImageUpdateContainer) GetEnvs() []v1.EnvVar { - env := make([]v1.EnvVar, 0) +func (a *ArangoDImageUpdateContainer) GetEnvs() []core.EnvVar { + env := make([]core.EnvVar, 0) if a.spec.License.HasSecretName() { env = append(env, k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, @@ -243,15 +248,23 @@ func (a *ArangoDImageUpdateContainer) GetEnvs() []v1.EnvVar { return nil } -func (a *ArangoDImageUpdateContainer) GetLifecycle() (*v1.Lifecycle, error) { +func (a *ArangoDImageUpdateContainer) GetLifecycle() (*core.Lifecycle, error) { return nil, nil } -func (a *ArangoDImageUpdateContainer) GetImagePullPolicy() v1.PullPolicy { +func (a *ArangoDImageUpdateContainer) GetImagePullPolicy() core.PullPolicy { return a.spec.GetImagePullPolicy() } -func (i *ImageUpdatePod) Init(pod *v1.Pod) { +func (i *ImageUpdatePod) GetName() string { + return i.apiObject.GetName() +} + +func (i *ImageUpdatePod) GetRole() string { + return "id" +} + +func (i *ImageUpdatePod) Init(pod *core.Pod) { terminationGracePeriodSeconds := int64((time.Second * 30).Seconds()) pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds } @@ -271,9 +284,9 @@ func (i *ImageUpdatePod) GetAffinityRole() string { return "" } -func (i *ImageUpdatePod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { - var volumes []v1.Volume - var volumeMounts []v1.VolumeMount +func (i *ImageUpdatePod) GetVolumes() ([]core.Volume, []core.VolumeMount) { + var volumes []core.Volume + var volumeMounts []core.VolumeMount volumes = append(volumes, k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount()) @@ -281,10 +294,10 @@ func (i *ImageUpdatePod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { return volumes, volumeMounts } -func (i *ImageUpdatePod) GetSidecars(*v1.Pod) { +func (i *ImageUpdatePod) GetSidecars(*core.Pod) { } -func (i *ImageUpdatePod) GetInitContainers() ([]v1.Container, error) { +func (i *ImageUpdatePod) GetInitContainers() ([]core.Container, error) { return nil, nil } @@ -292,14 +305,14 @@ func (i *ImageUpdatePod) GetFinalizers() []string { return nil } -func (i *ImageUpdatePod) GetTolerations() []v1.Toleration { +func (i *ImageUpdatePod) GetTolerations() []core.Toleration { shortDur := k8sutil.TolerationDuration{ Forever: false, TimeSpan: time.Second * 5, } - tolerations := make([]v1.Toleration, 0, 2) + tolerations := make([]core.Toleration, 0, 2) tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeNotReady, shortDur)) tolerations = k8sutil.AddTolerationIfNotFound(tolerations, @@ -322,8 +335,28 @@ func (i *ImageUpdatePod) GetServiceAccountName() string { return "" } -func (a *ArangoDImageUpdateContainer) GetSecurityContext() *v1.SecurityContext { +func (a *ArangoDImageUpdateContainer) GetSecurityContext() *core.SecurityContext { // Default security context var v api.ServerGroupSpecSecurityContext return v.NewSecurityContext() } + +func (i *ImageUpdatePod) GetPodAntiAffinity() *core.PodAntiAffinity { + a := core.PodAntiAffinity{} + + pod.AppendPodAntiAffinityDefault(i, &a) + + return pod.ReturnPodAntiAffinityOrNil(a) +} + +func (i *ImageUpdatePod) GetPodAffinity() *core.PodAffinity { + return nil +} + +func (i *ImageUpdatePod) GetNodeAffinity() *core.NodeAffinity { + a := core.NodeAffinity{} + + pod.AppendNodeSelector(&a) + + return pod.ReturnNodeAffinityOrNil(a) +} diff --git a/pkg/deployment/pod/affinity.go b/pkg/deployment/pod/affinity.go new file mode 100644 index 000000000..ae4bbff82 --- /dev/null +++ b/pkg/deployment/pod/affinity.go @@ -0,0 +1,131 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package pod + +import ( + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func AppendPodAntiAffinityDefault(p k8sutil.PodCreator, a *core.PodAntiAffinity) { + labels := k8sutil.LabelsForDeployment(p.GetName(), p.GetRole()) + labelSelector := &meta.LabelSelector{ + MatchLabels: labels, + } + + if !p.IsDeploymentMode() { + a.RequiredDuringSchedulingIgnoredDuringExecution = append(a.RequiredDuringSchedulingIgnoredDuringExecution, core.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: k8sutil.TopologyKeyHostname, + }) + } else { + a.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PreferredDuringSchedulingIgnoredDuringExecution, core.WeightedPodAffinityTerm{ + Weight: 1, + PodAffinityTerm: core.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: k8sutil.TopologyKeyHostname, + }, + }) + } +} + +func AppendNodeSelector(a *core.NodeAffinity) { + if a.RequiredDuringSchedulingIgnoredDuringExecution == nil { + a.RequiredDuringSchedulingIgnoredDuringExecution = &core.NodeSelector{} + } + + a.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(a.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, core.NodeSelectorTerm{ + MatchExpressions: []core.NodeSelectorRequirement{ + { + Key: "beta.kubernetes.io/arch", + Operator: "In", + Values: []string{"amd64"}, + }, + }, + }) +} + +func AppendAffinityWithRole(p k8sutil.PodCreator, a *core.PodAffinity, role string) { + labelSelector := &meta.LabelSelector{ + MatchLabels: k8sutil.LabelsForDeployment(p.GetName(), role), + } + if !p.IsDeploymentMode() { + a.RequiredDuringSchedulingIgnoredDuringExecution = append(a.RequiredDuringSchedulingIgnoredDuringExecution, core.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: k8sutil.TopologyKeyHostname, + }) + } else { + a.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PreferredDuringSchedulingIgnoredDuringExecution, core.WeightedPodAffinityTerm{ + Weight: 1, + PodAffinityTerm: core.PodAffinityTerm{ + LabelSelector: labelSelector, + TopologyKey: k8sutil.TopologyKeyHostname, + }, + }) + } +} + +func MergePodAntiAffinity(a ,b *core.PodAntiAffinity) { + if a == nil || b == nil { + return + } + + for _, rule := range b.PreferredDuringSchedulingIgnoredDuringExecution { + a.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PreferredDuringSchedulingIgnoredDuringExecution, rule) + } + + for _, rule := range b.RequiredDuringSchedulingIgnoredDuringExecution { + a.RequiredDuringSchedulingIgnoredDuringExecution = append(a.RequiredDuringSchedulingIgnoredDuringExecution, rule) + } +} + +func ReturnPodAffinityOrNil(a core.PodAffinity) *core.PodAffinity{ + if len(a.RequiredDuringSchedulingIgnoredDuringExecution) > 0 || len(a.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { + return &a + } + + return nil +} + +func ReturnPodAntiAffinityOrNil(a core.PodAntiAffinity) *core.PodAntiAffinity { + if len(a.RequiredDuringSchedulingIgnoredDuringExecution) > 0 || len(a.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { + return &a + } + + return nil +} + +func ReturnNodeAffinityOrNil( a core.NodeAffinity) *core.NodeAffinity { + if len(a.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { + return &a + } + + if s := a.RequiredDuringSchedulingIgnoredDuringExecution; s != nil { + if len(s.NodeSelectorTerms) > 0 { + return &a + } + } + + return nil +} \ No newline at end of file diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 88c460d9d..f94e5ed5e 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -661,9 +661,12 @@ func RenderArangoPod(deployment k8sutil.APIObject, role, id, podName string, p.Spec.Containers = append(p.Spec.Containers, c) podCreator.GetSidecars(&p) - // Add (anti-)affinity - p.Spec.Affinity = k8sutil.CreateAffinity(deployment.GetName(), role, !podCreator.IsDeploymentMode(), - podCreator.GetAffinityRole()) + // Add affinity + p.Spec.Affinity = &core.Affinity{ + NodeAffinity: podCreator.GetNodeAffinity(), + PodAntiAffinity: podCreator.GetPodAntiAffinity(), + PodAffinity: podCreator.GetPodAffinity(), + } return &p, nil } diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index 7e02bc4c3..f7e614fc7 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -24,6 +24,7 @@ package resources import ( "fmt" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "math" "github.com/arangodb/kube-arangodb/pkg/util" @@ -40,6 +41,8 @@ const ( ArangoDBOverrideDetectedTotalMemoryEnv = "ARANGODB_OVERRIDE_DETECTED_TOTAL_MEMORY" ) +var _ k8sutil.PodCreator = &MemberArangoDPod{} + type MemberArangoDPod struct { status api.MemberStatus tlsKeyfileSecretName string @@ -48,7 +51,6 @@ type MemberArangoDPod struct { groupSpec api.ServerGroupSpec spec api.DeploymentSpec group api.ServerGroup - context Context resources *Resources imageInfo api.ImageInfo } @@ -161,12 +163,38 @@ func (m *MemberArangoDPod) Init(pod *core.Pod) { pod.Spec.PriorityClassName = m.groupSpec.PriorityClassName } +func (m *MemberArangoDPod) GetName() string { + return m.resources.context.GetAPIObject().GetName() +} + +func (m *MemberArangoDPod) GetRole() string { + return m.group.AsRole() +} + func (m *MemberArangoDPod) GetImagePullSecrets() []string { return m.spec.ImagePullSecrets } -func (m *MemberArangoDPod) GetAffinityRole() string { - return "" +func (m *MemberArangoDPod) GetPodAntiAffinity() *core.PodAntiAffinity { + a := core.PodAntiAffinity{} + + pod.AppendPodAntiAffinityDefault(m, &a) + + pod.MergePodAntiAffinity(&a, m.groupSpec.AntiAffinity) + + return pod.ReturnPodAntiAffinityOrNil(a) +} + +func (m *MemberArangoDPod) GetPodAffinity() *core.PodAffinity { + return nil +} + +func (m *MemberArangoDPod) GetNodeAffinity() *core.NodeAffinity { + a := core.NodeAffinity{} + + pod.AppendNodeSelector(&a) + + return pod.ReturnNodeAffinityOrNil(a) } func (m *MemberArangoDPod) GetNodeSelector() map[string]string { diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index e418c6f51..3240dfcab 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -23,13 +23,14 @@ package resources import ( + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "math" "github.com/arangodb/kube-arangodb/pkg/util/constants" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" ) const ( @@ -44,6 +45,8 @@ type ArangoSyncContainer struct { imageInfo api.ImageInfo } +var _ k8sutil.PodCreator = &MemberSyncPod{} + type MemberSyncPod struct { tlsKeyfileSecretName string clientAuthCASecretName string @@ -60,12 +63,12 @@ func (a *ArangoSyncContainer) GetExecutor() string { return ArangoSyncExecutor } -func (a *ArangoSyncContainer) GetSecurityContext() *v1.SecurityContext { +func (a *ArangoSyncContainer) GetSecurityContext() *core.SecurityContext { return a.groupSpec.SecurityContext.NewSecurityContext() } -func (a *ArangoSyncContainer) GetProbes() (*v1.Probe, *v1.Probe, error) { - var liveness, readiness *v1.Probe +func (a *ArangoSyncContainer) GetProbes() (*core.Probe, *core.Probe, error) { + var liveness, readiness *core.Probe probeLivenessConfig, err := a.resources.getLivenessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion) if err != nil { @@ -88,18 +91,18 @@ func (a *ArangoSyncContainer) GetProbes() (*v1.Probe, *v1.Probe, error) { return liveness, readiness, nil } -func (a *ArangoSyncContainer) GetResourceRequirements() v1.ResourceRequirements { +func (a *ArangoSyncContainer) GetResourceRequirements() core.ResourceRequirements { return k8sutil.ExtractPodResourceRequirement(a.groupSpec.Resources) } -func (a *ArangoSyncContainer) GetLifecycle() (*v1.Lifecycle, error) { +func (a *ArangoSyncContainer) GetLifecycle() (*core.Lifecycle, error) { if a.resources.context.GetLifecycleImage() != "" { return k8sutil.NewLifecycle() } return nil, nil } -func (a *ArangoSyncContainer) GetImagePullPolicy() v1.PullPolicy { +func (a *ArangoSyncContainer) GetImagePullPolicy() core.PullPolicy { return a.spec.GetImagePullPolicy() } @@ -107,8 +110,8 @@ func (a *ArangoSyncContainer) GetImage() string { return a.imageInfo.Image } -func (a *ArangoSyncContainer) GetEnvs() []v1.EnvVar { - envs := make([]v1.EnvVar, 0) +func (a *ArangoSyncContainer) GetEnvs() []core.EnvVar { + envs := make([]core.EnvVar, 0) if a.spec.Sync.Monitoring.GetTokenSecretName() != "" { env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoSyncMonitoringToken, @@ -135,17 +138,46 @@ func (a *ArangoSyncContainer) GetEnvs() []v1.EnvVar { return nil } -func (m *MemberSyncPod) GetAffinityRole() string { - if m.group == api.ServerGroupSyncWorkers { - return api.ServerGroupDBServers.AsRole() - } - return "" +func (m *MemberSyncPod) GetName() string { + return m.resources.context.GetAPIObject().GetName() +} + +func (m *MemberSyncPod) GetRole() string { + return m.group.AsRole() } func (m *MemberSyncPod) GetImagePullSecrets() []string { return m.spec.ImagePullSecrets } +func (m *MemberSyncPod) GetPodAntiAffinity() *core.PodAntiAffinity { + a := core.PodAntiAffinity{} + + pod.AppendPodAntiAffinityDefault(m, &a) + + pod.MergePodAntiAffinity(&a, m.groupSpec.AntiAffinity) + + return pod.ReturnPodAntiAffinityOrNil(a) +} + +func (m *MemberSyncPod) GetPodAffinity() *core.PodAffinity { + a := core.PodAffinity{} + + if m.group == api.ServerGroupSyncWorkers { + pod.AppendAffinityWithRole(m, &a, api.ServerGroupDBServers.AsRole()) + } + + return pod.ReturnPodAffinityOrNil(a) +} + +func (m *MemberSyncPod) GetNodeAffinity() *core.NodeAffinity { + a := core.NodeAffinity{} + + pod.AppendNodeSelector(&a) + + return pod.ReturnNodeAffinityOrNil(a) +} + func (m *MemberSyncPod) GetNodeSelector() map[string]string { return m.groupSpec.GetNodeSelector() } @@ -154,7 +186,7 @@ func (m *MemberSyncPod) GetServiceAccountName() string { return m.groupSpec.GetServiceAccountName() } -func (m *MemberSyncPod) GetSidecars(pod *v1.Pod) { +func (m *MemberSyncPod) GetSidecars(pod *core.Pod) { // A sidecar provided by the user sidecars := m.groupSpec.GetSidecars() if len(sidecars) > 0 { @@ -162,9 +194,9 @@ func (m *MemberSyncPod) GetSidecars(pod *v1.Pod) { } } -func (m *MemberSyncPod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { - var volumes []v1.Volume - var volumeMounts []v1.VolumeMount +func (m *MemberSyncPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { + var volumes []core.Volume + var volumeMounts []core.VolumeMount if m.resources.context.GetLifecycleImage() != "" { volumes = append(volumes, k8sutil.LifecycleVolume()) @@ -205,8 +237,8 @@ func (m *MemberSyncPod) IsDeploymentMode() bool { return m.spec.IsDevelopment() } -func (m *MemberSyncPod) GetInitContainers() ([]v1.Container, error) { - var initContainers []v1.Container +func (m *MemberSyncPod) GetInitContainers() ([]core.Container, error) { + var initContainers []core.Container lifecycleImage := m.resources.context.GetLifecycleImage() if lifecycleImage != "" { @@ -225,7 +257,7 @@ func (m *MemberSyncPod) GetFinalizers() []string { return nil } -func (m *MemberSyncPod) GetTolerations() []v1.Toleration { +func (m *MemberSyncPod) GetTolerations() []core.Toleration { return m.resources.CreatePodTolerations(m.group, m.groupSpec) } @@ -239,7 +271,7 @@ func (m *MemberSyncPod) GetContainerCreator() k8sutil.ContainerCreator { } } -func (m *MemberSyncPod) Init(pod *v1.Pod) { +func (m *MemberSyncPod) Init(pod *core.Pod) { terminationGracePeriodSeconds := int64(math.Ceil(m.group.DefaultTerminationGracePeriod().Seconds())) pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds pod.Spec.PriorityClassName = m.groupSpec.PriorityClassName diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 433b5ae54..bef12a67b 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -34,7 +34,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" - v1 "k8s.io/api/core/v1" + core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) @@ -59,15 +59,19 @@ const ( ) type PodCreator interface { - Init(*v1.Pod) - GetVolumes() ([]v1.Volume, []v1.VolumeMount) - GetSidecars(*v1.Pod) - GetInitContainers() ([]v1.Container, error) + Init(*core.Pod) + GetName() string + GetRole() string + GetVolumes() ([]core.Volume, []core.VolumeMount) + GetSidecars(*core.Pod) + GetInitContainers() ([]core.Container, error) GetFinalizers() []string - GetTolerations() []v1.Toleration + GetTolerations() []core.Toleration GetNodeSelector() map[string]string GetServiceAccountName() string - GetAffinityRole() string + GetPodAntiAffinity() *core.PodAntiAffinity + GetPodAffinity() *core.PodAffinity + GetNodeAffinity() *core.NodeAffinity GetContainerCreator() ContainerCreator GetImagePullSecrets() []string IsDeploymentMode() bool @@ -75,37 +79,37 @@ type PodCreator interface { type ContainerCreator interface { GetExecutor() string - GetProbes() (*v1.Probe, *v1.Probe, error) - GetResourceRequirements() v1.ResourceRequirements - GetLifecycle() (*v1.Lifecycle, error) - GetImagePullPolicy() v1.PullPolicy + GetProbes() (*core.Probe, *core.Probe, error) + GetResourceRequirements() core.ResourceRequirements + GetLifecycle() (*core.Lifecycle, error) + GetImagePullPolicy() core.PullPolicy GetImage() string - GetEnvs() []v1.EnvVar - GetSecurityContext() *v1.SecurityContext + GetEnvs() []core.EnvVar + GetSecurityContext() *core.SecurityContext } // IsPodReady returns true if the PodReady condition on // the given pod is set to true. -func IsPodReady(pod *v1.Pod) bool { - condition := getPodCondition(&pod.Status, v1.PodReady) - return condition != nil && condition.Status == v1.ConditionTrue +func IsPodReady(pod *core.Pod) bool { + condition := getPodCondition(&pod.Status, core.PodReady) + return condition != nil && condition.Status == core.ConditionTrue } // GetPodByName returns pod if it exists among the pods' list // Returns false if not found. -func GetPodByName(pods []v1.Pod, podName string) (v1.Pod, bool) { +func GetPodByName(pods []core.Pod, podName string) (core.Pod, bool) { for _, pod := range pods { if pod.GetName() == podName { return pod, true } } - return v1.Pod{}, false + return core.Pod{}, false } // IsPodSucceeded returns true if the arangodb container of the pod // has terminated with exit code 0. -func IsPodSucceeded(pod *v1.Pod) bool { - if pod.Status.Phase == v1.PodSucceeded { +func IsPodSucceeded(pod *core.Pod) bool { + if pod.Status.Phase == core.PodSucceeded { return true } else { for _, c := range pod.Status.ContainerStatuses { @@ -124,8 +128,8 @@ func IsPodSucceeded(pod *v1.Pod) bool { // IsPodFailed returns true if the arangodb container of the pod // has terminated wih a non-zero exit code. -func IsPodFailed(pod *v1.Pod) bool { - if pod.Status.Phase == v1.PodFailed { +func IsPodFailed(pod *core.Pod) bool { + if pod.Status.Phase == core.PodFailed { return true } else { for _, c := range pod.Status.ContainerStatuses { @@ -144,40 +148,40 @@ func IsPodFailed(pod *v1.Pod) bool { } // IsPodScheduled returns true if the pod has been scheduled. -func IsPodScheduled(pod *v1.Pod) bool { - condition := getPodCondition(&pod.Status, v1.PodScheduled) - return condition != nil && condition.Status == v1.ConditionTrue +func IsPodScheduled(pod *core.Pod) bool { + condition := getPodCondition(&pod.Status, core.PodScheduled) + return condition != nil && condition.Status == core.ConditionTrue } // IsPodNotScheduledFor returns true if the pod has not been scheduled // for longer than the given duration. -func IsPodNotScheduledFor(pod *v1.Pod, timeout time.Duration) bool { - condition := getPodCondition(&pod.Status, v1.PodScheduled) +func IsPodNotScheduledFor(pod *core.Pod, timeout time.Duration) bool { + condition := getPodCondition(&pod.Status, core.PodScheduled) return condition != nil && - condition.Status == v1.ConditionFalse && + condition.Status == core.ConditionFalse && condition.LastTransitionTime.Time.Add(timeout).Before(time.Now()) } // IsPodMarkedForDeletion returns true if the pod has been marked for deletion. -func IsPodMarkedForDeletion(pod *v1.Pod) bool { +func IsPodMarkedForDeletion(pod *core.Pod) bool { return pod.DeletionTimestamp != nil } // IsPodTerminating returns true if the pod has been marked for deletion // but is still running. -func IsPodTerminating(pod *v1.Pod) bool { - return IsPodMarkedForDeletion(pod) && pod.Status.Phase == v1.PodRunning +func IsPodTerminating(pod *core.Pod) bool { + return IsPodMarkedForDeletion(pod) && pod.Status.Phase == core.PodRunning } // IsArangoDBImageIDAndVersionPod returns true if the given pod is used for fetching image ID and ArangoDB version of an image -func IsArangoDBImageIDAndVersionPod(p v1.Pod) bool { +func IsArangoDBImageIDAndVersionPod(p core.Pod) bool { role, found := p.GetLabels()[LabelKeyRole] return found && role == ImageIDAndVersionRole } // getPodCondition returns the condition of given type in the given status. // If not found, nil is returned. -func getPodCondition(status *v1.PodStatus, condType v1.PodConditionType) *v1.PodCondition { +func getPodCondition(status *core.PodStatus, condType core.PodConditionType) *core.PodCondition { for i := range status.Conditions { if status.Conditions[i].Type == condType { return &status.Conditions[i] @@ -208,62 +212,62 @@ func CreateTLSKeyfileSecretName(deploymentName, role, id string) string { } // ArangodVolumeMount creates a volume mount structure for arangod. -func ArangodVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func ArangodVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: ArangodVolumeName, MountPath: ArangodVolumeMountDir, } } // TlsKeyfileVolumeMount creates a volume mount structure for a TLS keyfile. -func TlsKeyfileVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func TlsKeyfileVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: TlsKeyfileVolumeName, MountPath: TLSKeyfileVolumeMountDir, } } // ClientAuthCACertificateVolumeMount creates a volume mount structure for a client-auth CA certificate (ca.crt). -func ClientAuthCACertificateVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func ClientAuthCACertificateVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: ClientAuthCAVolumeName, MountPath: ClientAuthCAVolumeMountDir, } } // MasterJWTVolumeMount creates a volume mount structure for a master JWT secret (token). -func MasterJWTVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func MasterJWTVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: MasterJWTSecretVolumeName, MountPath: MasterJWTSecretVolumeMountDir, } } // ClusterJWTVolumeMount creates a volume mount structure for a cluster JWT secret (token). -func ClusterJWTVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func ClusterJWTVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: ClusterJWTSecretVolumeName, MountPath: ClusterJWTSecretVolumeMountDir, } } -func ExporterJWTVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func ExporterJWTVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: ExporterJWTVolumeName, MountPath: ExporterJWTVolumeMountDir, } } // RocksdbEncryptionVolumeMount creates a volume mount structure for a RocksDB encryption key. -func RocksdbEncryptionVolumeMount() v1.VolumeMount { - return v1.VolumeMount{ +func RocksdbEncryptionVolumeMount() core.VolumeMount { + return core.VolumeMount{ Name: RocksdbEncryptionVolumeName, MountPath: RocksDBEncryptionVolumeMountDir, } } // ArangodInitContainer creates a container configured to initalize a UUID file. -func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool, securityContext *v1.SecurityContext) v1.Container { +func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool, securityContext *core.SecurityContext) core.Container { uuidFile := filepath.Join(ArangodVolumeMountDir, "UUID") engineFile := filepath.Join(ArangodVolumeMountDir, "ENGINE") var command string @@ -280,7 +284,7 @@ func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool } else { command = fmt.Sprintf("test -f %s || echo '%s' > %s", uuidFile, id, uuidFile) } - c := v1.Container{ + c := core.Container{ Name: name, Image: alpineImage, Command: []string{ @@ -288,17 +292,17 @@ func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool "-c", command, }, - Resources: v1.ResourceRequirements{ - Requests: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("10Mi"), + Resources: core.ResourceRequirements{ + Requests: core.ResourceList{ + core.ResourceCPU: resource.MustParse("100m"), + core.ResourceMemory: resource.MustParse("10Mi"), }, - Limits: v1.ResourceList{ - v1.ResourceCPU: resource.MustParse("100m"), - v1.ResourceMemory: resource.MustParse("50Mi"), + Limits: core.ResourceList{ + core.ResourceCPU: resource.MustParse("100m"), + core.ResourceMemory: resource.MustParse("50Mi"), }, }, - VolumeMounts: []v1.VolumeMount{ + VolumeMounts: []core.VolumeMount{ ArangodVolumeMount(), }, SecurityContext: securityContext, @@ -307,47 +311,47 @@ func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool } // ExtractPodResourceRequirement filters resource requirements for Pods. -func ExtractPodResourceRequirement(resources v1.ResourceRequirements) v1.ResourceRequirements { +func ExtractPodResourceRequirement(resources core.ResourceRequirements) core.ResourceRequirements { - filterStorage := func(list v1.ResourceList) v1.ResourceList { - newlist := make(v1.ResourceList) - if q, ok := list[v1.ResourceCPU]; ok { - newlist[v1.ResourceCPU] = q + filterStorage := func(list core.ResourceList) core.ResourceList { + newlist := make(core.ResourceList) + if q, ok := list[core.ResourceCPU]; ok { + newlist[core.ResourceCPU] = q } - if q, ok := list[v1.ResourceMemory]; ok { - newlist[v1.ResourceMemory] = q + if q, ok := list[core.ResourceMemory]; ok { + newlist[core.ResourceMemory] = q } return newlist } - return v1.ResourceRequirements{ + return core.ResourceRequirements{ Limits: filterStorage(resources.Limits), Requests: filterStorage(resources.Requests), } } // NewContainer creates a container for specified creator -func NewContainer(args []string, containerCreator ContainerCreator) (v1.Container, error) { +func NewContainer(args []string, containerCreator ContainerCreator) (core.Container, error) { liveness, readiness, err := containerCreator.GetProbes() if err != nil { - return v1.Container{}, err + return core.Container{}, err } lifecycle, err := containerCreator.GetLifecycle() if err != nil { - return v1.Container{}, err + return core.Container{}, err } - return v1.Container{ + return core.Container{ Name: ServerContainerName, Image: containerCreator.GetImage(), Command: append([]string{containerCreator.GetExecutor()}, args...), - Ports: []v1.ContainerPort{ + Ports: []core.ContainerPort{ { Name: "server", ContainerPort: int32(ArangoPort), - Protocol: v1.ProtocolTCP, + Protocol: core.ProtocolTCP, }, }, Env: containerCreator.GetEnvs(), @@ -361,19 +365,19 @@ func NewContainer(args []string, containerCreator ContainerCreator) (v1.Containe } // NewPod creates a basic Pod for given settings. -func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) v1.Pod { +func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) core.Pod { hostname := CreatePodHostName(deploymentName, role, id) - p := v1.Pod{ + p := core.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: LabelsForDeployment(deploymentName, role), Finalizers: podCreator.GetFinalizers(), }, - Spec: v1.PodSpec{ + Spec: core.PodSpec{ Hostname: hostname, Subdomain: CreateHeadlessServiceName(deploymentName), - RestartPolicy: v1.RestartPolicyNever, + RestartPolicy: core.RestartPolicyNever, Tolerations: podCreator.GetTolerations(), ServiceAccountName: podCreator.GetServiceAccountName(), NodeSelector: podCreator.GetNodeSelector(), @@ -383,9 +387,9 @@ func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) v1. // Add ImagePullSecrets imagePullSecrets := podCreator.GetImagePullSecrets() if imagePullSecrets != nil { - imagePullSecretsReference := make([]v1.LocalObjectReference, len(imagePullSecrets)) + imagePullSecretsReference := make([]core.LocalObjectReference, len(imagePullSecrets)) for id := range imagePullSecrets { - imagePullSecretsReference[id] = v1.LocalObjectReference{ + imagePullSecretsReference[id] = core.LocalObjectReference{ Name: imagePullSecrets[id], } } @@ -396,7 +400,7 @@ func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) v1. } // GetPodSpecChecksum return checksum of requested pod spec -func GetPodSpecChecksum(podSpec v1.PodSpec) (string, error) { +func GetPodSpecChecksum(podSpec core.PodSpec) (string, error) { // Do not calculate init containers podSpec.InitContainers = nil @@ -411,7 +415,7 @@ func GetPodSpecChecksum(podSpec v1.PodSpec) (string, error) { // CreatePod adds an owner to the given pod and calls the k8s api-server to created it. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePod(kubecli kubernetes.Interface, pod *v1.Pod, ns string, owner metav1.OwnerReference) (types.UID, string, error) { +func CreatePod(kubecli kubernetes.Interface, pod *core.Pod, ns string, owner metav1.OwnerReference) (types.UID, string, error) { addOwnerRefToObject(pod.GetObjectMeta(), &owner) checksum, err := GetPodSpecChecksum(pod.Spec) @@ -426,55 +430,55 @@ func CreatePod(kubecli kubernetes.Interface, pod *v1.Pod, ns string, owner metav } } -func CreateVolumeEmptyDir(name string) v1.Volume { - return v1.Volume{ +func CreateVolumeEmptyDir(name string) core.Volume { + return core.Volume{ Name: name, - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, + VolumeSource: core.VolumeSource{ + EmptyDir: &core.EmptyDirVolumeSource{}, }, } } -func CreateVolumeWithSecret(name, secretName string) v1.Volume { - return v1.Volume{ +func CreateVolumeWithSecret(name, secretName string) core.Volume { + return core.Volume{ Name: name, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ + VolumeSource: core.VolumeSource{ + Secret: &core.SecretVolumeSource{ SecretName: secretName, }, }, } } -func CreateVolumeWithPersitantVolumeClaim(name, claimName string) v1.Volume { - return v1.Volume{ +func CreateVolumeWithPersitantVolumeClaim(name, claimName string) core.Volume { + return core.Volume{ Name: name, - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + VolumeSource: core.VolumeSource{ + PersistentVolumeClaim: &core.PersistentVolumeClaimVolumeSource{ ClaimName: claimName, }, }, } } -func CreateEnvFieldPath(name, fieldPath string) v1.EnvVar { - return v1.EnvVar{ +func CreateEnvFieldPath(name, fieldPath string) core.EnvVar { + return core.EnvVar{ Name: name, - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ + ValueFrom: &core.EnvVarSource{ + FieldRef: &core.ObjectFieldSelector{ FieldPath: fieldPath, }, }, } } -func CreateEnvSecretKeySelector(name, SecretKeyName, secretKey string) v1.EnvVar { - return v1.EnvVar{ +func CreateEnvSecretKeySelector(name, SecretKeyName, secretKey string) core.EnvVar { + return core.EnvVar{ Name: name, Value: "", - ValueFrom: &v1.EnvVarSource{ - SecretKeyRef: &v1.SecretKeySelector{ - LocalObjectReference: v1.LocalObjectReference{ + ValueFrom: &core.EnvVarSource{ + SecretKeyRef: &core.SecretKeySelector{ + LocalObjectReference: core.LocalObjectReference{ Name: SecretKeyName, }, Key: secretKey, From c7e087996957640ffcb0cf62ce44cf201ac7ca1d Mon Sep 17 00:00:00 2001 From: ajanikow Date: Fri, 20 Mar 2020 14:45:51 +0000 Subject: [PATCH 2/6] fmt --- pkg/deployment/deployment_affinity_test.go | 10 +++++----- pkg/deployment/deployment_run_test.go | 7 ++++--- pkg/deployment/images.go | 11 ++++++----- pkg/deployment/pod/affinity.go | 8 ++++---- pkg/deployment/resources/pod_creator.go | 4 ++-- pkg/deployment/resources/pod_creator_arangod.go | 3 ++- pkg/deployment/resources/pod_creator_sync.go | 3 ++- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/pkg/deployment/deployment_affinity_test.go b/pkg/deployment/deployment_affinity_test.go index 18cfc9b6e..2e47e13bb 100644 --- a/pkg/deployment/deployment_affinity_test.go +++ b/pkg/deployment/deployment_affinity_test.go @@ -31,7 +31,7 @@ import ( core "k8s.io/api/core/v1" ) -func modifyAffinity(name, group string, required bool, role string, mods ... func(a *core.Affinity)) *core.Affinity { +func modifyAffinity(name, group string, required bool, role string, mods ...func(a *core.Affinity)) *core.Affinity { affinity := k8sutil.CreateAffinity(name, group, required, role) @@ -61,7 +61,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { Authentication: noAuthentication, TLS: noTLS, DBServers: api.ServerGroupSpec{ - AntiAffinity:&core.PodAntiAffinity{ + AntiAffinity: &core.PodAntiAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ testAffinity, }, @@ -123,7 +123,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { Authentication: noAuthentication, TLS: noTLS, DBServers: api.ServerGroupSpec{ - AntiAffinity:&core.PodAntiAffinity{ + AntiAffinity: &core.PodAntiAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ weight, }, @@ -185,7 +185,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { Authentication: noAuthentication, TLS: noTLS, DBServers: api.ServerGroupSpec{ - AntiAffinity:&core.PodAntiAffinity{ + AntiAffinity: &core.PodAntiAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ weight, }, @@ -251,7 +251,7 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { Authentication: noAuthentication, TLS: noTLS, DBServers: api.ServerGroupSpec{ - AntiAffinity:&core.PodAntiAffinity{ + AntiAffinity: &core.PodAntiAffinity{ PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ weight, weight, diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index 1a9db65e4..eab0fc2f1 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -25,10 +25,11 @@ package deployment import ( "encoding/json" "fmt" + "testing" + "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" core "k8s.io/api/core/v1" - "testing" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/stretchr/testify/assert" @@ -77,7 +78,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { if util.BoolOrDefault(testCase.CompareChecksum, true) { compareSpec(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) } - require.Equal(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) + require.Equal(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) require.Equal(t, testCase.ExpectedPod.ObjectMeta, pods.Items[0].ObjectMeta) if len(testCase.ExpectedEvent) > 0 { @@ -129,4 +130,4 @@ func compareSpec(t *testing.T, a, b core.PodSpec) { require.Equal(t, string(aj), string(bj)) require.Equal(t, ac, bc) -} \ No newline at end of file +} diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 4fd7a900e..e3c500575 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -26,10 +26,11 @@ import ( "context" "crypto/sha1" "fmt" - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "strings" "time" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/rs/zerolog" core "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -45,9 +46,9 @@ import ( var _ k8sutil.PodCreator = &ImageUpdatePod{} type ImageUpdatePod struct { - spec api.DeploymentSpec + spec api.DeploymentSpec apiObject k8sutil.APIObject - image string + image string } type ArangoDImageUpdateContainer struct { @@ -195,8 +196,8 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima } imagePod := ImageUpdatePod{ - spec: ib.Spec, - image: image, + spec: ib.Spec, + image: image, apiObject: ib.APIObject, } diff --git a/pkg/deployment/pod/affinity.go b/pkg/deployment/pod/affinity.go index ae4bbff82..aa348cf0c 100644 --- a/pkg/deployment/pod/affinity.go +++ b/pkg/deployment/pod/affinity.go @@ -86,7 +86,7 @@ func AppendAffinityWithRole(p k8sutil.PodCreator, a *core.PodAffinity, role stri } } -func MergePodAntiAffinity(a ,b *core.PodAntiAffinity) { +func MergePodAntiAffinity(a, b *core.PodAntiAffinity) { if a == nil || b == nil { return } @@ -100,7 +100,7 @@ func MergePodAntiAffinity(a ,b *core.PodAntiAffinity) { } } -func ReturnPodAffinityOrNil(a core.PodAffinity) *core.PodAffinity{ +func ReturnPodAffinityOrNil(a core.PodAffinity) *core.PodAffinity { if len(a.RequiredDuringSchedulingIgnoredDuringExecution) > 0 || len(a.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { return &a } @@ -116,7 +116,7 @@ func ReturnPodAntiAffinityOrNil(a core.PodAntiAffinity) *core.PodAntiAffinity { return nil } -func ReturnNodeAffinityOrNil( a core.NodeAffinity) *core.NodeAffinity { +func ReturnNodeAffinityOrNil(a core.NodeAffinity) *core.NodeAffinity { if len(a.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { return &a } @@ -128,4 +128,4 @@ func ReturnNodeAffinityOrNil( a core.NodeAffinity) *core.NodeAffinity { } return nil -} \ No newline at end of file +} diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index f94e5ed5e..2820c9475 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -663,9 +663,9 @@ func RenderArangoPod(deployment k8sutil.APIObject, role, id, podName string, // Add affinity p.Spec.Affinity = &core.Affinity{ - NodeAffinity: podCreator.GetNodeAffinity(), + NodeAffinity: podCreator.GetNodeAffinity(), PodAntiAffinity: podCreator.GetPodAntiAffinity(), - PodAffinity: podCreator.GetPodAffinity(), + PodAffinity: podCreator.GetPodAffinity(), } return &p, nil diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index f7e614fc7..d73c780f9 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -24,9 +24,10 @@ package resources import ( "fmt" - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "math" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/constants" diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 3240dfcab..40085a432 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -23,9 +23,10 @@ package resources import ( - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" "math" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util/constants" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" From 6440345b48720652a9fe8d099cf9471f401e7a6a Mon Sep 17 00:00:00 2001 From: ajanikow Date: Mon, 23 Mar 2020 06:36:37 +0000 Subject: [PATCH 3/6] Add Affinity and NodeAffinity --- pkg/apis/deployment/v1/server_group_spec.go | 4 + pkg/deployment/deployment_affinity_test.go | 374 +++++++++++++++++- pkg/deployment/pod/affinity.go | 34 ++ .../resources/pod_creator_arangod.go | 8 +- pkg/deployment/resources/pod_creator_sync.go | 4 + 5 files changed, 422 insertions(+), 2 deletions(-) diff --git a/pkg/apis/deployment/v1/server_group_spec.go b/pkg/apis/deployment/v1/server_group_spec.go index 45fd1368d..5c793d3b7 100644 --- a/pkg/apis/deployment/v1/server_group_spec.go +++ b/pkg/apis/deployment/v1/server_group_spec.go @@ -72,6 +72,10 @@ type ServerGroupSpec struct { VolumeResizeMode *PVCResizeMode `json:"pvcResizeMode,omitempty"` // AntiAffinity specified additional antiAffinity settings in ArangoDB Pod definitions AntiAffinity *v1.PodAntiAffinity `json:"antiAffinity,omitempty"` + // Affinity specified additional affinity settings in ArangoDB Pod definitions + Affinity *v1.PodAffinity `json:"affinity,omitempty"` + // NodeAffinity specified additional nodeAffinity settings in ArangoDB Pod definitions + NodeAffinity *v1.NodeAffinity `json:"nodeAffinity,omitempty"` // Sidecars specifies a list of additional containers to be started Sidecars []v1.Container `json:"sidecars,omitempty"` // SecurityContext specifies security context for group diff --git a/pkg/deployment/deployment_affinity_test.go b/pkg/deployment/deployment_affinity_test.go index 2e47e13bb..2ca5ff9e9 100644 --- a/pkg/deployment/deployment_affinity_test.go +++ b/pkg/deployment/deployment_affinity_test.go @@ -42,7 +42,7 @@ func modifyAffinity(name, group string, required bool, role string, mods ...func return affinity } -func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { +func TestEnsurePod_ArangoDB_AntiAffinity(t *testing.T) { testAffinity := core.PodAffinityTerm{ TopologyKey: "myTopologyKey", } @@ -317,3 +317,375 @@ func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { runTestCases(t, testCases...) } + +func TestEnsurePod_ArangoDB_Affinity(t *testing.T) { + testAffinity := core.PodAffinityTerm{ + TopologyKey: "myTopologyKey", + } + + weight := core.WeightedPodAffinityTerm{ + Weight: 6, + PodAffinityTerm: testAffinity, + } + + testCases := []testCaseStruct{ + { + Name: "DBserver POD with affinity required", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + Affinity: &core.PodAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ + testAffinity, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + if a.PodAffinity == nil { + a.PodAffinity = &core.PodAffinity{} + } + a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, testAffinity) + }), + }, + }, + }, + { + Name: "DBserver POD with affinity prefered", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + Affinity: &core.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + weight, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + if a.PodAffinity == nil { + a.PodAffinity = &core.PodAffinity{} + } + a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, weight) + }), + }, + }, + }, + { + Name: "DBserver POD with affinity both", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + Affinity: &core.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + weight, + }, + RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ + testAffinity, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + if a.PodAffinity == nil { + a.PodAffinity = &core.PodAffinity{} + } + a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, weight) + a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, testAffinity) + }), + }, + }, + }, + { + Name: "DBserver POD with affinity mixed", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + Affinity: &core.PodAffinity{ + PreferredDuringSchedulingIgnoredDuringExecution: []core.WeightedPodAffinityTerm{ + weight, + weight, + weight, + weight, + }, + RequiredDuringSchedulingIgnoredDuringExecution: []core.PodAffinityTerm{ + testAffinity, + testAffinity, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + if a.PodAffinity == nil { + a.PodAffinity = &core.PodAffinity{} + } + a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, weight, weight, weight, weight) + a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution = append(a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, testAffinity, testAffinity) + }), + }, + }, + }, + } + + runTestCases(t, testCases...) +} + +func TestEnsurePod_ArangoDB_NodeAffinity(t *testing.T) { + testCases := []testCaseStruct{ + { + Name: "DBserver POD with nodeAffinity required", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + NodeAffinity: &core.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &core.NodeSelector{ + NodeSelectorTerms: []core.NodeSelectorTerm{ + { + MatchFields: []core.NodeSelectorRequirement{ + { + Key: "key", + }, + }, + }, + }, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: emptyResources, + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: modifyAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, "", func(a *core.Affinity) { + n := core.NodeSelectorTerm{ + MatchFields: []core.NodeSelectorRequirement{ + { + Key: "key", + }, + }, + } + a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(a.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, n) + }), + }, + }, + }, + } + + runTestCases(t, testCases...) +} diff --git a/pkg/deployment/pod/affinity.go b/pkg/deployment/pod/affinity.go index aa348cf0c..290d224fe 100644 --- a/pkg/deployment/pod/affinity.go +++ b/pkg/deployment/pod/affinity.go @@ -100,6 +100,40 @@ func MergePodAntiAffinity(a, b *core.PodAntiAffinity) { } } +func MergePodAffinity(a, b *core.PodAffinity) { + if a == nil || b == nil { + return + } + + for _, rule := range b.PreferredDuringSchedulingIgnoredDuringExecution { + a.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PreferredDuringSchedulingIgnoredDuringExecution, rule) + } + + for _, rule := range b.RequiredDuringSchedulingIgnoredDuringExecution { + a.RequiredDuringSchedulingIgnoredDuringExecution = append(a.RequiredDuringSchedulingIgnoredDuringExecution, rule) + } +} + +func MergeNodeAffinity(a, b *core.NodeAffinity) { + if a == nil || b == nil { + return + } + + for _, rule := range b.PreferredDuringSchedulingIgnoredDuringExecution { + a.PreferredDuringSchedulingIgnoredDuringExecution = append(a.PreferredDuringSchedulingIgnoredDuringExecution, rule) + } + + if b.RequiredDuringSchedulingIgnoredDuringExecution != nil { + if a.RequiredDuringSchedulingIgnoredDuringExecution == nil { + a.RequiredDuringSchedulingIgnoredDuringExecution = b.RequiredDuringSchedulingIgnoredDuringExecution.DeepCopy() + } else { + for _, rule := range b.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms { + a.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append(a.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, rule) + } + } + } +} + func ReturnPodAffinityOrNil(a core.PodAffinity) *core.PodAffinity { if len(a.RequiredDuringSchedulingIgnoredDuringExecution) > 0 || len(a.PreferredDuringSchedulingIgnoredDuringExecution) > 0 { return &a diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index d73c780f9..afd654e53 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -187,7 +187,11 @@ func (m *MemberArangoDPod) GetPodAntiAffinity() *core.PodAntiAffinity { } func (m *MemberArangoDPod) GetPodAffinity() *core.PodAffinity { - return nil + a := core.PodAffinity{} + + pod.MergePodAffinity(&a, m.groupSpec.Affinity) + + return pod.ReturnPodAffinityOrNil(a) } func (m *MemberArangoDPod) GetNodeAffinity() *core.NodeAffinity { @@ -195,6 +199,8 @@ func (m *MemberArangoDPod) GetNodeAffinity() *core.NodeAffinity { pod.AppendNodeSelector(&a) + pod.MergeNodeAffinity(&a, m.groupSpec.NodeAffinity) + return pod.ReturnNodeAffinityOrNil(a) } diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 40085a432..3db0486c9 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -168,6 +168,8 @@ func (m *MemberSyncPod) GetPodAffinity() *core.PodAffinity { pod.AppendAffinityWithRole(m, &a, api.ServerGroupDBServers.AsRole()) } + pod.MergePodAffinity(&a, m.groupSpec.Affinity) + return pod.ReturnPodAffinityOrNil(a) } @@ -176,6 +178,8 @@ func (m *MemberSyncPod) GetNodeAffinity() *core.NodeAffinity { pod.AppendNodeSelector(&a) + pod.MergeNodeAffinity(&a, m.groupSpec.NodeAffinity) + return pod.ReturnNodeAffinityOrNil(a) } From e778b2d4069111257284792820795f7e2908f1e2 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Mon, 23 Mar 2020 07:14:31 +0000 Subject: [PATCH 4/6] Update changelog --- CHANGELOG.md | 3 ++- pkg/apis/deployment/v1/zz_generated.deepcopy.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index df9e81d77..e4ad8f15a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Change Log -## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) +## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A +- Added Customizable Affinity settings for ArangoDB Member Pods) - Added possibility to override default images used by ArangoDeployment - Added possibility to set probes on all groups - Added Image Discovery type in ArangoDeployment spec diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index c2a1770e8..7709c6205 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -965,6 +965,21 @@ func (in *ServerGroupSpec) DeepCopyInto(out *ServerGroupSpec) { *out = new(PVCResizeMode) **out = **in } + if in.AntiAffinity != nil { + in, out := &in.AntiAffinity, &out.AntiAffinity + *out = new(corev1.PodAntiAffinity) + (*in).DeepCopyInto(*out) + } + if in.Affinity != nil { + in, out := &in.Affinity, &out.Affinity + *out = new(corev1.PodAffinity) + (*in).DeepCopyInto(*out) + } + if in.NodeAffinity != nil { + in, out := &in.NodeAffinity, &out.NodeAffinity + *out = new(corev1.NodeAffinity) + (*in).DeepCopyInto(*out) + } if in.Sidecars != nil { in, out := &in.Sidecars, &out.Sidecars *out = make([]corev1.Container, len(*in)) From 68e058173a32abf82fcfbed71769f662e65b8519 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Mon, 23 Mar 2020 21:55:59 +0000 Subject: [PATCH 5/6] Fix Unit Tests --- pkg/deployment/resources/pod_creator_arangod.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index afd654e53..8703b1ce7 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -52,6 +52,7 @@ type MemberArangoDPod struct { groupSpec api.ServerGroupSpec spec api.DeploymentSpec group api.ServerGroup + context Context resources *Resources imageInfo api.ImageInfo } From 1d722f0eee1aad459b7d64775b18bd07e06dd131 Mon Sep 17 00:00:00 2001 From: ajanikow Date: Mon, 23 Mar 2020 22:02:33 +0000 Subject: [PATCH 6/6] Update CHANGELOG --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4ad8f15a..e55d24a14 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Change Log -## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A -- Added Customizable Affinity settings for ArangoDB Member Pods) +## [master](https://github.com/arangodb/kube-arangodb/tree/master) (N/A) +- Added Customizable Affinity settings for ArangoDB Member Pods - Added possibility to override default images used by ArangoDeployment - Added possibility to set probes on all groups - Added Image Discovery type in ArangoDeployment spec