From e1e302895ae4ae4b2a017479a43593cb88836a0c Mon Sep 17 00:00:00 2001 From: informalict Date: Thu, 14 Nov 2019 15:54:36 +0100 Subject: [PATCH 1/3] Move sources into proper places Function CreateArangodPod has been cleaned Clean up CreateArangoSyncPod function Create interface for new containers Improvements Seperate logic into apart files Fix Nil pointer exception First unit test Add first unit test for EnsurePods function verify TLS in unit tests Add 9 unit tests for EnsurePods function Cleaning code Adjust unit test to master branch Unit test for arangosync pod Fix unit tests for sync master according to branch master Move creating volumes and volume mounts into one place Add unit test for sync worker Add unit tests for ensureImage function Add unit tests for update image add imagepullsecret and secretpullpolicy to interface Add image and env to interface Cleaning Add new unit tests Check deployment cleaning the code --- pkg/deployment/deployment_test.go | 2327 +++++++++++++++++ pkg/deployment/images.go | 149 +- pkg/deployment/images_test.go | 363 +++ pkg/deployment/resources/certificates_tls.go | 4 +- pkg/deployment/resources/pod_creator.go | 153 +- .../resources/pod_creator_arangod.go | 261 ++ pkg/deployment/resources/pod_creator_sync.go | 208 ++ pkg/deployment/resources/pod_inspector.go | 2 +- pkg/util/k8sutil/affinity.go | 4 +- pkg/util/k8sutil/affinity_test.go | 9 +- pkg/util/k8sutil/lifecycle.go | 80 + pkg/util/k8sutil/pods.go | 640 ++--- reboot.go | 9 +- 13 files changed, 3614 insertions(+), 595 deletions(-) create mode 100644 pkg/deployment/deployment_test.go create mode 100644 pkg/deployment/images_test.go create mode 100644 pkg/deployment/resources/pod_creator_arangod.go create mode 100644 pkg/deployment/resources/pod_creator_sync.go create mode 100644 pkg/util/k8sutil/lifecycle.go diff --git a/pkg/deployment/deployment_test.go b/pkg/deployment/deployment_test.go new file mode 100644 index 000000000..8bec168ba --- /dev/null +++ b/pkg/deployment/deployment_test.go @@ -0,0 +1,2327 @@ +package deployment + +import ( + "io/ioutil" + "os" + "testing" + + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + "github.com/pkg/errors" + + "github.com/arangodb/go-driver/jwt" + + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + + "github.com/stretchr/testify/assert" + + "github.com/stretchr/testify/require" + + "github.com/arangodb/kube-arangodb/pkg/util" + + "github.com/rs/zerolog" + + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + recordfake "k8s.io/client-go/tools/record" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + arangofake "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/fake" +) + +const ( + testNamespace = "default" + testDeploymentName = "test" + testVersion = "3.5.2" + testImage = "arangodb/arangodb:" + testVersion + testCASecretName = "testCA" + testJWTSecretName = "testJWT" + testExporterToken = "testExporterToken" + testRocksDBEncryptionKey = "testRocksDB" + testPersistentVolumeClaimName = "testClaim" + testLicense = "testLicense" + testServiceAccountName = "testServiceAccountName" + testPriorityClassName = "testPriority" + testImageLifecycle = "arangodb/kube-arangodb:0.3.16" + testImageAlpine = "alpine:3.7" +) + +type testCaseStruct struct { + Name string + ArangoDeployment *api.ArangoDeployment + Helper func(*testing.T, *Deployment, *testCaseStruct) + config Config + ExpectedError error + ExpectedEvent string + ExpectedPod v1.Pod +} + +func TestEnsurePods(t *testing.T) { + // Arange + defaultAgentTerminationTimeout := int64(api.ServerGroupAgents.DefaultTerminationGracePeriod().Seconds()) + defaultDBServerTerminationTimeout := int64(api.ServerGroupDBServers.DefaultTerminationGracePeriod().Seconds()) + defaultCoordinatorTerminationTimeout := int64(api.ServerGroupCoordinators.DefaultTerminationGracePeriod().Seconds()) + defaultSingleTerminationTimeout := int64(api.ServerGroupSingle.DefaultTerminationGracePeriod().Seconds()) + defaultSyncMasterTerminationTimeout := int64(api.ServerGroupSyncMasters.DefaultTerminationGracePeriod().Seconds()) + defaultSyncWorkerTerminationTimeout := int64(api.ServerGroupSyncWorkers.DefaultTerminationGracePeriod().Seconds()) + + nodeSelectorTest := map[string]string{ + "test": "test", + } + + firstAgentStatus := api.MemberStatus{ + ID: "agent1", + Phase: api.MemberPhaseNone, + } + + firstCoordinatorStatus := api.MemberStatus{ + ID: "coordinator1", + Phase: api.MemberPhaseNone, + } + + singleStatus := api.MemberStatus{ + ID: "single1", + Phase: api.MemberPhaseNone, + } + + firstSyncMaster := api.MemberStatus{ + ID: "syncMaster1", + Phase: api.MemberPhaseNone, + } + + firstSyncWorker := api.MemberStatus{ + ID: "syncWorker1", + Phase: api.MemberPhaseNone, + } + + firstDBServerStatus := api.MemberStatus{ + ID: "DBserver1", + Phase: api.MemberPhaseNone, + } + + noAuthentication := api.AuthenticationSpec{ + JWTSecretName: util.NewString(api.JWTSecretNameDisabled), + } + + noTLS := api.TLSSpec{ + CASecretName: util.NewString(api.CASecretNameDisabled), + } + + authenticationSpec := api.AuthenticationSpec{ + JWTSecretName: util.NewString(testJWTSecretName), + } + tlsSpec := api.TLSSpec{ + CASecretName: util.NewString(testCASecretName), + } + + rocksDBSpec := api.RocksDBSpec{ + Encryption: api.RocksDBEncryptionSpec{ + KeySecretName: util.NewString(testRocksDBEncryptionKey), + }, + } + + metricsSpec := api.MetricsSpec{ + Enabled: util.NewBool(true), + Image: util.NewString("arangodb/arangodb-exporter:0.1.6"), + Authentication: api.MetricsAuthenticationSpec{ + JWTTokenSecretName: util.NewString(testExporterToken), + }, + } + + resourcesUnfiltered := v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("500m"), + v1.ResourceMemory: resource.MustParse("2Gi"), + v1.ResourceStorage: resource.MustParse("8Gi"), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: resource.MustParse("100m"), + v1.ResourceMemory: resource.MustParse("1Gi"), + v1.ResourceStorage: resource.MustParse("2Gi"), + }, + } + + sidecarName1 := "sidecar1" + sidecarName2 := "sidecar2" + + testCases := []testCaseStruct{ + { + Name: "Agent Pod with image pull policy", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + ImagePullPolicy: util.NewPullPolicy(v1.PullAlways), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullAlways, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with sidecar", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Agents: api.ServerGroupSpec{ + Sidecars: []v1.Container{ + { + Name: sidecarName1, + }, + { + Name: sidecarName2, + }, + }, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + { + Name: sidecarName1, + }, + { + Name: sidecarName2, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with image pull secrets", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + ImagePullSecrets: []string{"docker-registry", "other-registry"}, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + ImagePullSecrets: []v1.LocalObjectReference{ + { + Name: "docker-registry", + }, + { + Name: "other-registry", + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with alpine init container", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + }, + }, + config: Config{ + AlpineImage: testImageAlpine, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + InitContainers: []v1.Container{ + createTestAlpineContainer(firstAgentStatus.ID, false), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "DBserver POD with resource requirements", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + Resources: resourcesUnfiltered, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: k8sutil.ExtractPodResourceRequirement(resourcesUnfiltered), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver POD with resource requirements and persistent volume claim", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + DBServers: api.ServerGroupSpec{ + Resources: resourcesUnfiltered, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + deployment.status.last.Members.DBServers[0].PersistentVolumeClaimName = testPersistentVolumeClaimName + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName, + testPersistentVolumeClaimName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + Resources: k8sutil.ExtractPodResourceRequirement(resourcesUnfiltered), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "Initialized DBserver POD with alpine init container", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + }, + }, + config: Config{ + AlpineImage: testImageAlpine, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + deployment.status.last.Members.DBServers[0].IsInitialized = true + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + InitContainers: []v1.Container{ + createTestAlpineContainer(firstDBServerStatus.ID, true), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod without TLS, authentication, persistent volume claim, metrics, rocksDB encryption, license", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with persistent volume claim", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + agentWithPersistentVolumeClaim := firstAgentStatus + agentWithPersistentVolumeClaim.PersistentVolumeClaimName = testPersistentVolumeClaimName + + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + agentWithPersistentVolumeClaim, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName, + testPersistentVolumeClaimName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with TLS", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: tlsSpec, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupAgentsString, firstAgentStatus.ID), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, true, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(true, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with authentication and unsecured liveness", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + TLS: noTLS, + Environment: api.NewEnvironment(api.EnvironmentProduction), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + + authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(false, + authorization, k8sutil.ArangoPort) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, true, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.ClusterJWTVolumeMount(), + }, + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + true, ""), + }, + }, + }, + { + Name: "Agent Pod with TLS and authentication and secured liveness probe", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + TLS: api.TLSSpec{ + CASecretName: util.NewString(testCASecretName), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + + authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + authorization, k8sutil.ArangoPort) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupAgentsString, firstAgentStatus.ID), + k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, true, true, false), + Ports: createTestPorts(), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + k8sutil.ClusterJWTVolumeMount(), + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod with encrypted rocksdb", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + RocksDB: rocksDBSpec, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + + secrets := deployment.GetKubeCli().CoreV1().Secrets(testNamespace) + key := make([]byte, 32) + k8sutil.CreateEncryptionKeySecret(secrets, testRocksDBEncryptionKey, key) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, testRocksDBEncryptionKey), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, true), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.RocksdbEncryptionVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "Agent Pod can not have metrics exporter", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: metricsSpec, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Agents: api.MemberStatusList{ + firstAgentStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupAgents, firstAgentStatus) + }, + ExpectedEvent: "member agent is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForAgent(firstAgentStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultAgentTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupAgentsString + "-" + firstAgentStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupAgentsString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: metricsSpec, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + { + Name: k8sutil.ExporterContainerName, + Image: "arangodb/arangodb-exporter:0.1.6", + Command: createTestExporterCommand(false), + Ports: createTestExporterPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ExporterJWTVolumeMount(), + }, + LivenessProbe: createTestExporterLivenessProbe(false), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter and lifecycle init container", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: metricsSpec, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + }, + config: Config{ + LifecycleImage: testImageLifecycle, + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + k8sutil.LifecycleVolume(), + }, + InitContainers: []v1.Container{ + createTestLifecycleContainer(), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Env: []v1.EnvVar{ + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"), + }, + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.LifecycleVolumeMounts(), + }, + Lifecycle: createTestLifecycle(), + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + { + Name: k8sutil.ExporterContainerName, + Image: "arangodb/arangodb-exporter:0.1.6", + Command: createTestExporterCommand(false), + Ports: createTestExporterPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ExporterJWTVolumeMount(), + }, + LivenessProbe: createTestExporterLivenessProbe(false), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter and lifecycle init container and alpine init container", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: noTLS, + Metrics: metricsSpec, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + }, + config: Config{ + LifecycleImage: testImageLifecycle, + AlpineImage: testImageAlpine, + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + k8sutil.LifecycleVolume(), + }, + InitContainers: []v1.Container{ + createTestLifecycleContainer(), + createTestAlpineContainer(firstDBServerStatus.ID, false), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, false, false, false), + Env: []v1.EnvVar{ + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"), + }, + Ports: createTestPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.LifecycleVolumeMounts(), + }, + Lifecycle: createTestLifecycle(), + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + { + Name: k8sutil.ExporterContainerName, + Image: "arangodb/arangodb-exporter:0.1.6", + Command: createTestExporterCommand(false), + Ports: createTestExporterPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ExporterJWTVolumeMount(), + }, + LivenessProbe: createTestExporterLivenessProbe(false), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + { + Name: "DBserver Pod with metrics exporter, lifecycle, tls, authentication, license, rocksDB encryption, secured liveness", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + TLS: tlsSpec, + Metrics: metricsSpec, + RocksDB: rocksDBSpec, + Environment: api.NewEnvironment(api.EnvironmentProduction), + License: api.LicenseSpec{ + SecretName: util.NewString(testLicense), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + testCase.ExpectedPod.ObjectMeta.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + + secrets := deployment.GetKubeCli().CoreV1().Secrets(testNamespace) + key := make([]byte, 32) + k8sutil.CreateEncryptionKeySecret(secrets, testRocksDBEncryptionKey, key) + + authorization, err := createTestToken(deployment, testCase, []string{"/_api/version"}) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + authorization, k8sutil.ArangoPort) + }, + config: Config{ + LifecycleImage: testImageLifecycle, + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupDBServersString, firstDBServerStatus.ID), + k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, testRocksDBEncryptionKey), + k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, testExporterToken), + k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName), + k8sutil.LifecycleVolume(), + }, + InitContainers: []v1.Container{ + createTestLifecycleContainer(), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForDBServer(firstDBServerStatus.ID, true, true, true), + Env: []v1.EnvVar{ + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, + testLicense, constants.SecretKeyToken), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"), + }, + Ports: createTestPorts(), + Lifecycle: createTestLifecycle(), + LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.LifecycleVolumeMounts(), + k8sutil.TlsKeyfileVolumeMount(), + k8sutil.RocksdbEncryptionVolumeMount(), + k8sutil.ClusterJWTVolumeMount(), + }, + }, + { + Name: k8sutil.ExporterContainerName, + Image: "arangodb/arangodb-exporter:0.1.6", + Command: createTestExporterCommand(true), + Ports: createTestExporterPorts(), + VolumeMounts: []v1.VolumeMount{ + k8sutil.ExporterJWTVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + }, + LivenessProbe: createTestExporterLivenessProbe(true), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + true, ""), + }, + }, + }, + { + Name: "Coordinator Pod with TLS and authentication and readiness and liveness", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + Environment: api.NewEnvironment(api.EnvironmentProduction), + TLS: api.TLSSpec{ + CASecretName: util.NewString(testCASecretName), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Coordinators: api.MemberStatusList{ + firstCoordinatorStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupCoordinators, firstCoordinatorStatus) + + auth, err := createTestToken(deployment, testCase, []string{"/_admin/server/availability"}) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(true, auth) + }, + ExpectedEvent: "member coordinator is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupCoordinatorsString, firstCoordinatorStatus.ID), + k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForCoordinator(firstCoordinatorStatus.ID, true, true, false), + Ports: createTestPorts(), + ImagePullPolicy: v1.PullIfNotPresent, + Resources: v1.ResourceRequirements{ + Limits: make(v1.ResourceList), + Requests: make(v1.ResourceList), + }, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + k8sutil.ClusterJWTVolumeMount(), + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultCoordinatorTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupCoordinatorsString + "-" + firstCoordinatorStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupCoordinatorsString, + true, ""), + }, + }, + }, + { + Name: "Single Pod with TLS and authentication and readiness and readiness", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + TLS: api.TLSSpec{ + CASecretName: util.NewString(testCASecretName), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Single: api.MemberStatusList{ + singleStatus, + }, + }, + Images: createTestImages(false), + } + + testCase.createTestPodData(deployment, api.ServerGroupSingle, singleStatus) + + authLiveness, err := createTestToken(deployment, testCase, []string{"/_api/version"}) + require.NoError(t, err) + + authReadiness, err := createTestToken(deployment, testCase, []string{"/_admin/server/availability"}) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, + authLiveness, 0) + testCase.ExpectedPod.Spec.Containers[0].ReadinessProbe = createTestReadinessProbe(true, authReadiness) + }, + ExpectedEvent: "member single is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupSingleString, singleStatus.ID), + k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForSingleMode(singleStatus.ID, true, true, false), + Ports: createTestPorts(), + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + k8sutil.ClusterJWTVolumeMount(), + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultSingleTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupSingleString + "-" + singleStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupSingleString, + false, ""), + }, + }, + }, + //ArangoD container - end + + // Arango sync master container - start + { + Name: "Sync Pod does not work for enterprise image", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(false), + } + }, + ExpectedError: errors.New("Image '" + testImage + "' does not contain an Enterprise version of ArangoDB"), + }, + { + Name: "Sync Pod cannot get master JWT secret", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + }, + ExpectedError: errors.New("Master JWT secret validation failed: secrets \"" + + testDeploymentName + "-sync-jwt\" not found"), + }, + { + Name: "Sync Pod cannot get monitoring token secret", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + + secretName := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() + err := deployment.GetKubeCli().CoreV1().Secrets(testNamespace).Delete(secretName, &metav1.DeleteOptions{}) + require.NoError(t, err) + }, + ExpectedError: errors.New("Monitoring token secret validation failed: secrets \"" + + testDeploymentName + "-sync-mt\" not found"), + }, + { + Name: "Sync Master Pod cannot create TLS keyfile secret", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + + secretName := testCase.ArangoDeployment.Spec.Sync.TLS.GetCASecretName() + err := deployment.GetKubeCli().CoreV1().Secrets(testNamespace).Delete(secretName, &metav1.DeleteOptions{}) + require.NoError(t, err) + }, + ExpectedError: errors.New("Failed to create TLS keyfile secret: secrets \"" + + testDeploymentName + "-sync-ca\" not found"), + }, + { + Name: "Sync Master Pod cannot get cluster JWT secret", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + + secretName := testCase.ArangoDeployment.Spec.Authentication.GetJWTSecretName() + err := deployment.GetKubeCli().CoreV1().Secrets(testNamespace).Delete(secretName, &metav1.DeleteOptions{}) + require.NoError(t, err) + }, + ExpectedError: errors.New("Cluster JWT secret validation failed: secrets \"" + + testJWTSecretName + "\" not found"), + }, + { + Name: "Sync Master Pod cannot get authentication CA certificate", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + + secretName := testCase.ArangoDeployment.Spec.Sync.Authentication.GetClientCASecretName() + err := deployment.GetKubeCli().CoreV1().Secrets(testNamespace).Delete(secretName, &metav1.DeleteOptions{}) + require.NoError(t, err) + }, + ExpectedError: errors.New("Client authentication CA certificate secret validation failed: " + + "secrets \"" + testDeploymentName + "-sync-client-auth-ca\" not found"), + }, + { + Name: "Sync Master Pod with authentication, monitoring, tls, service account, node selector, " + + "liveness probe, priority class name, resource requirements", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: authenticationSpec, + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + SyncMasters: api.ServerGroupSpec{ + ServiceAccountName: util.NewString(testServiceAccountName), + NodeSelector: nodeSelectorTest, + PriorityClassName: testPriorityClassName, + Resources: resourcesUnfiltered, + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + + testCase.createTestPodData(deployment, api.ServerGroupSyncMasters, firstSyncMaster) + + name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() + auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( + true, "bearer "+auth, k8sutil.ArangoSyncMasterPort) + }, + ExpectedEvent: "member syncmaster is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + createTestTLSVolume(api.ServerGroupSyncMastersString, firstSyncMaster.ID), + k8sutil.CreateVolumeWithSecret(k8sutil.ClientAuthCAVolumeName, "test-sync-client-auth-ca"), + k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, "test-sync-jwt"), + k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, testJWTSecretName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForSyncMaster(firstSyncMaster.ID, true, true, true), + Ports: createTestPorts(), + Env: []v1.EnvVar{ + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoSyncMonitoringToken, + testDeploymentName+"-sync-mt", constants.SecretKeyToken), + }, + ImagePullPolicy: v1.PullIfNotPresent, + Resources: resourcesUnfiltered, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.TlsKeyfileVolumeMount(), + k8sutil.ClientAuthCACertificateVolumeMount(), + k8sutil.MasterJWTVolumeMount(), + k8sutil.ClusterJWTVolumeMount(), + }, + }, + }, + PriorityClassName: testPriorityClassName, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: testServiceAccountName, + NodeSelector: nodeSelectorTest, + TerminationGracePeriodSeconds: &defaultSyncMasterTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupSyncMastersString + "-" + + firstSyncMaster.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupSyncMastersString, + false, ""), + }, + }, + }, + { + Name: "Sync Master Pod with lifecycle, license, monitoring without authentication and alpine", + config: Config{ + LifecycleImage: testImageLifecycle, + AlpineImage: testImageAlpine, + }, + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + Environment: api.NewEnvironment(api.EnvironmentProduction), + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + License: api.LicenseSpec{ + SecretName: util.NewString(testLicense), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncMasters: api.MemberStatusList{ + firstSyncMaster, + }, + }, + Images: createTestImages(true), + } + + testCase.createTestPodData(deployment, api.ServerGroupSyncMasters, firstSyncMaster) + name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() + auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( + true, "bearer "+auth, k8sutil.ArangoSyncMasterPort) + }, + ExpectedEvent: "member syncmaster is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.LifecycleVolume(), + createTestTLSVolume(api.ServerGroupSyncMastersString, firstSyncMaster.ID), + k8sutil.CreateVolumeWithSecret(k8sutil.ClientAuthCAVolumeName, + testDeploymentName+"-sync-client-auth-ca"), + k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, + testDeploymentName+"-sync-jwt"), + }, + InitContainers: []v1.Container{ + createTestLifecycleContainer(), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForSyncMaster(firstSyncMaster.ID, true, false, true), + Ports: createTestPorts(), + Env: []v1.EnvVar{ + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoSyncMonitoringToken, + testDeploymentName+"-sync-mt", constants.SecretKeyToken), + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, + testLicense, constants.SecretKeyToken), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"), + }, + ImagePullPolicy: v1.PullIfNotPresent, + Lifecycle: createTestLifecycle(), + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.LifecycleVolumeMounts(), + k8sutil.TlsKeyfileVolumeMount(), + k8sutil.ClientAuthCACertificateVolumeMount(), + k8sutil.MasterJWTVolumeMount(), + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultSyncMasterTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupSyncMastersString + "-" + + firstSyncMaster.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupSyncMastersString, + true, ""), + }, + }, + }, + // Arango sync master container - end + + // Arango sync worker - start + { + Name: "Sync Worker Pod with monitoring, service account, node selector, lifecycle, license " + + "liveness probe, priority class name, resource requirements without alpine", + config: Config{ + LifecycleImage: testImageLifecycle, + AlpineImage: testImageAlpine, + }, + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + Sync: api.SyncSpec{ + Enabled: util.NewBool(true), + }, + SyncWorkers: api.ServerGroupSpec{ + ServiceAccountName: util.NewString(testServiceAccountName), + NodeSelector: nodeSelectorTest, + PriorityClassName: testPriorityClassName, + Resources: resourcesUnfiltered, + }, + License: api.LicenseSpec{ + SecretName: util.NewString(testLicense), + }, + }, + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + SyncWorkers: api.MemberStatusList{ + firstSyncWorker, + }, + }, + Images: createTestImages(true), + } + + testCase.createTestPodData(deployment, api.ServerGroupSyncWorkers, firstSyncWorker) + + name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() + auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + require.NoError(t, err) + + testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( + true, "bearer "+auth, k8sutil.ArangoSyncWorkerPort) + }, + ExpectedEvent: "member syncworker is created", + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.LifecycleVolume(), + k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, testDeploymentName+"-sync-jwt"), + }, + InitContainers: []v1.Container{ + createTestLifecycleContainer(), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForSyncWorker(firstSyncWorker.ID, true, true), + Ports: createTestPorts(), + Env: []v1.EnvVar{ + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoSyncMonitoringToken, + testDeploymentName+"-sync-mt", constants.SecretKeyToken), + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, + testLicense, constants.SecretKeyToken), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"), + k8sutil.CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"), + }, + Lifecycle: createTestLifecycle(), + ImagePullPolicy: v1.PullIfNotPresent, + Resources: resourcesUnfiltered, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.LifecycleVolumeMounts(), + k8sutil.MasterJWTVolumeMount(), + }, + }, + }, + PriorityClassName: testPriorityClassName, + RestartPolicy: v1.RestartPolicyNever, + ServiceAccountName: testServiceAccountName, + NodeSelector: nodeSelectorTest, + TerminationGracePeriodSeconds: &defaultSyncWorkerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupSyncWorkersString + "-" + + firstSyncWorker.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupSyncWorkersString, + false, api.ServerGroupDBServersString), + }, + }, + }, + // Arango sync worker - end + } + + for _, testCase := range testCases { + //nolint:scopelint + t.Run(testCase.Name, func(t *testing.T) { + // Arrange + d, eventRecorder := createTestDeployment(testCase.config, testCase.ArangoDeployment) + + err := d.resources.EnsureSecrets() + require.NoError(t, err) + + if testCase.Helper != nil { + testCase.Helper(t, d, &testCase) + } + + // Create custom resource in the fake kubernetes API + _, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(testNamespace).Create(d.apiObject) + require.NoError(t, err) + + // Act + err = d.resources.EnsurePods() + + // Assert + if testCase.ExpectedError != nil { + assert.EqualError(t, err, testCase.ExpectedError.Error()) + return + } + + require.NoError(t, err) + pods, err := d.deps.KubeCli.CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + require.Equal(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) + require.Equal(t, testCase.ExpectedPod.ObjectMeta, pods.Items[0].ObjectMeta) + + if len(testCase.ExpectedEvent) > 0 { + select { + case msg := <-eventRecorder.Events: + assert.Contains(t, msg, testCase.ExpectedEvent) + default: + assert.Fail(t, "expected event", "expected event with message '%s'", testCase.ExpectedEvent) + } + + status, version := d.GetStatus() + assert.Equal(t, int32(1), version) + + checkEachMember := func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error { + for _, m := range *status { + require.Equal(t, api.MemberPhaseCreated, m.Phase) + + _, exist := m.Conditions.Get(api.ConditionTypeReady) + require.Equal(t, false, exist) + _, exist = m.Conditions.Get(api.ConditionTypeTerminated) + require.Equal(t, false, exist) + _, exist = m.Conditions.Get(api.ConditionTypeTerminating) + require.Equal(t, false, exist) + _, exist = m.Conditions.Get(api.ConditionTypeAgentRecoveryNeeded) + require.Equal(t, false, exist) + _, exist = m.Conditions.Get(api.ConditionTypeAutoUpgrade) + require.Equal(t, false, exist) + } + return nil + } + + d.GetServerGroupIterator().ForeachServerGroup(checkEachMember, &status) + } + }) + } +} + +func createTestTLSVolume(serverGroupString, ID string) v1.Volume { + return k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, + k8sutil.CreateTLSKeyfileSecretName(testDeploymentName, serverGroupString, ID)) +} + +func createTestLifecycle() *v1.Lifecycle { + lifecycle, _ := k8sutil.NewLifecycle() + return lifecycle +} + +func createTestToken(deployment *Deployment, testCase *testCaseStruct, paths []string) (string, error) { + + name := testCase.ArangoDeployment.Spec.Authentication.GetJWTSecretName() + s, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + if err != nil { + return "", err + } + + return jwt.CreateArangodJwtAuthorizationHeaderAllowedPaths(s, "kube-arangodb", paths) +} + +func createTestLivenessProbe(secure bool, authorization string, port int) *v1.Probe { + return k8sutil.HTTPProbeConfig{ + LocalPath: "/_api/version", + Secure: secure, + Authorization: authorization, + Port: port, + }.Create() +} + +func createTestReadinessProbe(secure bool, authorization string) *v1.Probe { + return k8sutil.HTTPProbeConfig{ + LocalPath: "/_admin/server/availability", + Secure: secure, + Authorization: authorization, + InitialDelaySeconds: 2, + PeriodSeconds: 2, + }.Create() +} + +func createTestCommandForDBServer(name string, tls, auth, encryptionRocksDB bool) []string { + command := []string{resources.ArangoDExecutor} + if tls { + command = append(command, "--cluster.my-address=ssl://"+testDeploymentName+"-"+ + api.ServerGroupDBServersString+"-"+name+".test-int.default.svc:8529") + } else { + command = append(command, "--cluster.my-address=tcp://"+testDeploymentName+"-"+ + api.ServerGroupDBServersString+"-"+name+".test-int.default.svc:8529") + } + + command = append(command, "--cluster.my-role=PRIMARY", "--database.directory=/data", + "--foxx.queues=false", "--log.level=INFO", "--log.output=+") + + if encryptionRocksDB { + command = append(command, "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key") + } + + if auth { + command = append(command, "--server.authentication=true") + } else { + command = append(command, "--server.authentication=false") + } + + if tls { + command = append(command, "--server.endpoint=ssl://[::]:8529") + } else { + command = append(command, "--server.endpoint=tcp://[::]:8529") + } + + if auth { + command = append(command, "--server.jwt-secret-keyfile=/secrets/cluster/jwt/token") + } + + command = append(command, "--server.statistics=true", "--server.storage-engine=rocksdb") + + if tls { + command = append(command, "--ssl.ecdh-curve=", "--ssl.keyfile=/secrets/tls/tls.keyfile") + } + return command +} + +func createTestCommandForCoordinator(name string, tls, auth, encryptionRocksDB bool) []string { + command := []string{resources.ArangoDExecutor} + if tls { + command = append(command, "--cluster.my-address=ssl://"+testDeploymentName+"-"+ + api.ServerGroupCoordinatorsString+"-"+name+".test-int.default.svc:8529") + } else { + command = append(command, "--cluster.my-address=tcp://"+testDeploymentName+"-"+ + api.ServerGroupCoordinatorsString+"-"+name+".test-int.default.svc:8529") + } + + command = append(command, "--cluster.my-role=COORDINATOR", "--database.directory=/data", + "--foxx.queues=true", "--log.level=INFO", "--log.output=+") + + if encryptionRocksDB { + command = append(command, "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key") + } + + if auth { + command = append(command, "--server.authentication=true") + } else { + command = append(command, "--server.authentication=false") + } + + if tls { + command = append(command, "--server.endpoint=ssl://[::]:8529") + } else { + command = append(command, "--server.endpoint=tcp://[::]:8529") + } + + if auth { + command = append(command, "--server.jwt-secret-keyfile=/secrets/cluster/jwt/token") + } + + command = append(command, "--server.statistics=true", "--server.storage-engine=rocksdb") + + if tls { + command = append(command, "--ssl.ecdh-curve=", "--ssl.keyfile=/secrets/tls/tls.keyfile") + } + return command +} + +func createTestCommandForSingleMode(name string, tls, auth, encryptionRocksDB bool) []string { + command := []string{resources.ArangoDExecutor} + + command = append(command, "--database.directory=/data", "--foxx.queues=true", "--log.level=INFO", + "--log.output=+") + + if encryptionRocksDB { + command = append(command, "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key") + } + + if auth { + command = append(command, "--server.authentication=true") + } else { + command = append(command, "--server.authentication=false") + } + + if tls { + command = append(command, "--server.endpoint=ssl://[::]:8529") + } else { + command = append(command, "--server.endpoint=tcp://[::]:8529") + } + + if auth { + command = append(command, "--server.jwt-secret-keyfile=/secrets/cluster/jwt/token") + } + + command = append(command, "--server.statistics=true", "--server.storage-engine=rocksdb") + + if tls { + command = append(command, "--ssl.ecdh-curve=", "--ssl.keyfile=/secrets/tls/tls.keyfile") + } + return command +} + +func createTestCommandForAgent(name string, tls, auth, encryptionRocksDB bool) []string { + command := []string{ + resources.ArangoDExecutor, + "--agency.activate=true", + "--agency.disaster-recovery-id=" + name} + + if tls { + command = append(command, "--agency.my-address=ssl://"+testDeploymentName+"-"+ + api.ServerGroupAgentsString+"-"+name+"."+testDeploymentName+"-int."+testNamespace+".svc:8529") + } else { + command = append(command, "--agency.my-address=tcp://"+testDeploymentName+"-"+ + api.ServerGroupAgentsString+"-"+name+"."+testDeploymentName+"-int."+testNamespace+".svc:8529") + } + + command = append(command, + "--agency.size=3", + "--agency.supervision=true", + "--database.directory=/data", + "--foxx.queues=false", + "--log.level=INFO", + "--log.output=+") + + if encryptionRocksDB { + command = append(command, "--rocksdb.encryption-keyfile=/secrets/rocksdb/encryption/key") + } + + if auth { + command = append(command, "--server.authentication=true") + } else { + command = append(command, "--server.authentication=false") + } + + if tls { + command = append(command, "--server.endpoint=ssl://[::]:8529") + } else { + command = append(command, "--server.endpoint=tcp://[::]:8529") + } + + if auth { + command = append(command, "--server.jwt-secret-keyfile=/secrets/cluster/jwt/token") + } + + command = append(command, "--server.statistics=false", "--server.storage-engine=rocksdb") + + if tls { + command = append(command, "--ssl.ecdh-curve=", "--ssl.keyfile=/secrets/tls/tls.keyfile") + } + + return command +} + +func createTestCommandForSyncMaster(name string, tls, auth, monitoring bool) []string { + command := []string{resources.ArangoSyncExecutor, "run", "master"} + + if tls { + command = append(command, "--cluster.endpoint=https://"+testDeploymentName+":8529") + } else { + command = append(command, "--cluster.endpoint=http://"+testDeploymentName+":8529") + } + + if auth { + command = append(command, "--cluster.jwt-secret=/secrets/cluster/jwt/token") + } + + command = append(command, "--master.endpoint=https://"+testDeploymentName+"-sync.default.svc:8629") + + command = append(command, "--master.jwt-secret=/secrets/master/jwt/token") + + if monitoring { + command = append(command, "--monitoring.token="+"$("+constants.EnvArangoSyncMonitoringToken+")") + } + + command = append(command, "--mq.type=direct", "--server.client-cafile=/secrets/client-auth/ca/ca.crt") + + command = append(command, "--server.endpoint=https://"+testDeploymentName+ + "-syncmaster-"+name+".test-int."+testNamespace+".svc:8629", + "--server.keyfile=/secrets/tls/tls.keyfile", "--server.port=8629") + + return command +} + +func createTestCommandForSyncWorker(name string, tls, monitoring bool) []string { + command := []string{resources.ArangoSyncExecutor, "run", "worker"} + + scheme := "http" + if tls { + scheme = "https" + } + + command = append(command, + "--master.endpoint=https://"+testDeploymentName+"-sync:8629", + "--master.jwt-secret=/secrets/master/jwt/token") + + if monitoring { + command = append(command, "--monitoring.token="+"$("+constants.EnvArangoSyncMonitoringToken+")") + } + + command = append(command, + "--server.endpoint="+scheme+"://"+testDeploymentName+"-syncworker-"+name+".test-int."+testNamespace+".svc:8729", + "--server.port=8729") + + return command +} + +func createTestDeployment(config Config, arangoDeployment *api.ArangoDeployment) (*Deployment, *recordfake.FakeRecorder) { + + eventRecorder := recordfake.NewFakeRecorder(10) + kubernetesClientSet := fake.NewSimpleClientset() + + arangoDeployment.ObjectMeta = metav1.ObjectMeta{ + Name: testDeploymentName, + Namespace: testNamespace, + } + + deps := Dependencies{ + Log: zerolog.New(ioutil.Discard), + KubeCli: kubernetesClientSet, + DatabaseCRCli: arangofake.NewSimpleClientset(&api.ArangoDeployment{}), + EventRecorder: eventRecorder, + } + + d := &Deployment{ + apiObject: arangoDeployment, + config: config, + deps: deps, + eventCh: make(chan *deploymentEvent, deploymentEventQueueSize), + stopCh: make(chan struct{}), + clientCache: newClientCache(deps.KubeCli, arangoDeployment), + } + + arangoDeployment.Spec.SetDefaults(arangoDeployment.GetName()) + d.resources = resources.NewResources(deps.Log, d) + + return d, eventRecorder +} + +func createTestPorts() []v1.ContainerPort { + return []v1.ContainerPort{ + { + Name: "server", + ContainerPort: 8529, + Protocol: "TCP", + }, + } +} + +func createTestImages(enterprise bool) api.ImageInfoList { + return api.ImageInfoList{ + { + Image: testImage, + ArangoDBVersion: testVersion, + ImageID: testImage, + Enterprise: enterprise, + }, + } +} + +func createTestExporterPorts() []v1.ContainerPort { + return []v1.ContainerPort{ + { + Name: "exporter", + ContainerPort: 9101, + Protocol: "TCP", + }, + } +} + +func createTestExporterCommand(secure bool) []string { + command := []string{ + "/app/arangodb-exporter", + } + + if secure { + command = append(command, "--arangodb.endpoint=https://localhost:8529") + } else { + command = append(command, "--arangodb.endpoint=http://localhost:8529") + } + + command = append(command, "--arangodb.jwt-file=/secrets/exporter/jwt/token") + + if secure { + command = append(command, "--ssl.keyfile=/secrets/tls/tls.keyfile") + } + return command +} + +func createTestExporterLivenessProbe(secure bool) *v1.Probe { + return k8sutil.HTTPProbeConfig{ + LocalPath: "/", + Port: k8sutil.ArangoExporterPort, + Secure: secure, + }.Create() +} + +func createTestLifecycleContainer() v1.Container { + binaryPath, _ := os.Executable() + + return v1.Container{ + Name: "init-lifecycle", + Image: testImageLifecycle, + Command: []string{binaryPath, "lifecycle", "copy", "--target", "/lifecycle/tools"}, + VolumeMounts: []v1.VolumeMount{ + k8sutil.LifecycleVolumeMounts(), + }, + ImagePullPolicy: "IfNotPresent", + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + } +} + +func createTestAlpineContainer(name string, requireUUID bool) v1.Container { + return k8sutil.ArangodInitContainer("uuid", name, "rocksdb", testImageAlpine, requireUUID) +} + +func (testCase *testCaseStruct) createTestPodData(deployment *Deployment, group api.ServerGroup, + memberStatus api.MemberStatus) { + + podName := k8sutil.CreatePodName(testDeploymentName, group.AsRoleAbbreviated(), memberStatus.ID, + resources.CreatePodSuffix(testCase.ArangoDeployment.Spec)) + + testCase.ExpectedPod.ObjectMeta = metav1.ObjectMeta{ + Name: podName, + Namespace: testNamespace, + Labels: k8sutil.LabelsForDeployment(testDeploymentName, group.AsRole()), + OwnerReferences: []metav1.OwnerReference{ + testCase.ArangoDeployment.AsOwner(), + }, + Finalizers: deployment.resources.CreatePodFinalizers(group), + } + + groupSpec := testCase.ArangoDeployment.Spec.GetServerGroupSpec(group) + testCase.ExpectedPod.Spec.Tolerations = deployment.resources.CreatePodTolerations(group, groupSpec) +} diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 60f23af58..54c720c8f 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -35,14 +35,21 @@ import ( "k8s.io/client-go/kubernetes" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) -const ( - dockerPullableImageIDPrefix_ = "docker-pullable://" -) +type ImageUpdatePod struct { + spec api.DeploymentSpec + image string +} + +type ArangoDImageUpdateContainer struct { + spec api.DeploymentSpec + image string +} type imagesBuilder struct { APIObject k8sutil.APIObject @@ -182,26 +189,130 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima "--database.directory=" + k8sutil.ArangodVolumeMountDir, "--log.output=+", } - terminationGracePeriod := time.Second * 30 - tolerations := make([]v1.Toleration, 0, 2) - shortDur := k8sutil.TolerationDuration{Forever: false, TimeSpan: time.Second * 5} - tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeNotReady, shortDur)) - tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeUnreachable, shortDur)) - tolerations = k8sutil.AddTolerationIfNotFound(tolerations, k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeAlphaUnreachable, shortDur)) - serviceAccountName := "" - - env := make(map[string]k8sutil.EnvValue) - if ib.Spec.License.HasSecretName() { - env[constants.EnvArangoLicenseKey] = k8sutil.EnvValue{ - SecretName: ib.Spec.License.GetSecretName(), - SecretKey: constants.SecretKeyToken, - } + + imagePod := ImageUpdatePod{ + spec: ib.Spec, + image: image, } - if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, "", "", ib.Spec.GetImagePullPolicy(), ib.Spec.ImagePullSecrets, "", false, terminationGracePeriod, args, env, nil, nil, nil, - tolerations, serviceAccountName, "", "", "", nil, "", v1.ResourceRequirements{}, nil, nil, nil); err != nil { + + if err := resources.CreateArangoPod(ib.KubeCli, ib.APIObject, role, id, podName, args, &imagePod); err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, maskAny(err) } // Come back soon to inspect the pod return true, nil } + +func (a *ArangoDImageUpdateContainer) GetExecutor() string { + return resources.ArangoDExecutor +} + +func (a *ArangoDImageUpdateContainer) GetProbes() (*v1.Probe, *v1.Probe, error) { + return nil, nil, nil +} + +func (a *ArangoDImageUpdateContainer) GetResourceRequirements() v1.ResourceRequirements { + return v1.ResourceRequirements{ + Limits: make(v1.ResourceList), + Requests: make(v1.ResourceList), + } +} + +func (a *ArangoDImageUpdateContainer) GetImage() string { + return a.image +} + +func (a *ArangoDImageUpdateContainer) GetEnvs() []v1.EnvVar { + env := make([]v1.EnvVar, 0) + + if a.spec.License.HasSecretName() { + env = append(env, k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, + a.spec.License.GetSecretName(), constants.SecretKeyToken)) + } + + if len(env) > 0 { + return env + } + + return nil +} + +func (a *ArangoDImageUpdateContainer) GetLifecycle() (*v1.Lifecycle, error) { + return nil, nil +} + +func (a *ArangoDImageUpdateContainer) GetImagePullPolicy() v1.PullPolicy { + return a.spec.GetImagePullPolicy() +} + +func (i *ImageUpdatePod) Init(pod *v1.Pod) { + terminationGracePeriodSeconds := int64((time.Second * 30).Seconds()) + pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds +} + +func (i *ImageUpdatePod) GetImagePullSecrets() []string { + return i.spec.ImagePullSecrets +} + +func (i *ImageUpdatePod) GetContainerCreator() k8sutil.ContainerCreator { + return &ArangoDImageUpdateContainer{ + spec: i.spec, + image: i.image, + } +} + +func (i *ImageUpdatePod) GetAffinityRole() string { + return "" +} + +func (i *ImageUpdatePod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { + var volumes []v1.Volume + var volumeMounts []v1.VolumeMount + + volumes = append(volumes, k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) + volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount()) + + return volumes, volumeMounts +} + +func (i *ImageUpdatePod) GetSidecars(*v1.Pod) { + return +} + +func (i *ImageUpdatePod) GetInitContainers() ([]v1.Container, error) { + return nil, nil +} + +func (i *ImageUpdatePod) GetFinalizers() []string { + return nil +} + +func (i *ImageUpdatePod) GetTolerations() []v1.Toleration { + + shortDur := k8sutil.TolerationDuration{ + Forever: false, + TimeSpan: time.Second * 5, + } + + tolerations := make([]v1.Toleration, 0, 2) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, + k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeNotReady, shortDur)) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, + k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeUnreachable, shortDur)) + tolerations = k8sutil.AddTolerationIfNotFound(tolerations, + k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeAlphaUnreachable, shortDur)) + + return tolerations +} + +func (i *ImageUpdatePod) IsDeploymentMode() bool { + return true +} + +func (i *ImageUpdatePod) GetNodeSelector() map[string]string { + return nil +} + +func (i *ImageUpdatePod) GetServiceAccountName() string { + return "" +} diff --git a/pkg/deployment/images_test.go b/pkg/deployment/images_test.go new file mode 100644 index 000000000..6cc02a2d6 --- /dev/null +++ b/pkg/deployment/images_test.go @@ -0,0 +1,363 @@ +package deployment + +import ( + "crypto/sha1" + "fmt" + "testing" + "time" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + "github.com/arangodb/kube-arangodb/pkg/deployment/resources" + + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/assert" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/stretchr/testify/require" +) + +const ( + testNewImage = testImage + "2" +) + +type testCaseImageUpdate struct { + Name string + ArangoDeployment *api.ArangoDeployment + Before func(*testing.T, *Deployment) + After func(*testing.T, *Deployment) + ExpectedError error + RetrySoon bool + ExpectedPod v1.Pod +} + +func TestEnsureImages(t *testing.T) { + // Arange + terminationGracePeriodSeconds := int64((time.Second * 30).Seconds()) + id := fmt.Sprintf("%0x", sha1.Sum([]byte(testNewImage)))[:6] + hostname := testDeploymentName + "-" + k8sutil.ImageIDAndVersionRole + "-" + id + + testCases := []testCaseImageUpdate{ + { + Name: "Image has not been changed", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + }, + }, + }, + { + Name: "Image has been changed", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + }, + }, + RetrySoon: true, + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testNewImage, + Command: createTestCommandForImageUpdatePod(), + Ports: createTestPorts(), + Resources: v1.ResourceRequirements{ + Limits: make(v1.ResourceList), + Requests: make(v1.ResourceList), + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Tolerations: getTestTolerations(), + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Hostname: hostname, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, + k8sutil.ImageIDAndVersionRole, false, ""), + }, + }, + }, + { + Name: "Image not been changed with license", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + License: api.LicenseSpec{ + SecretName: util.NewString(testLicense), + }, + }, + }, + RetrySoon: true, + ExpectedPod: v1.Pod{ + Spec: v1.PodSpec{ + Volumes: []v1.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + }, + Containers: []v1.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testNewImage, + Command: createTestCommandForImageUpdatePod(), + Ports: createTestPorts(), + Env: []v1.EnvVar{ + k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, + testLicense, constants.SecretKeyToken), + }, + Resources: v1.ResourceRequirements{ + Limits: make(v1.ResourceList), + Requests: make(v1.ResourceList), + }, + VolumeMounts: []v1.VolumeMount{ + k8sutil.ArangodVolumeMount(), + }, + ImagePullPolicy: v1.PullIfNotPresent, + SecurityContext: &v1.SecurityContext{ + Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, + }, + }, + }, + RestartPolicy: v1.RestartPolicyNever, + Tolerations: getTestTolerations(), + TerminationGracePeriodSeconds: &terminationGracePeriodSeconds, + Hostname: hostname, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, + k8sutil.ImageIDAndVersionRole, false, ""), + }, + }, + }, + { + Name: "Image is being updated in failed phase", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + }, + }, + Before: func(t *testing.T, deployment *Deployment) { + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.CreatePodName(testDeploymentName, k8sutil.ImageIDAndVersionRole, id, ""), + CreationTimestamp: metav1.Now(), + }, + Spec: v1.PodSpec{}, + Status: v1.PodStatus{ + Phase: v1.PodFailed, + }, + } + + _, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).Create(&pod) + require.NoError(t, err) + }, + After: func(t *testing.T, deployment *Deployment) { + pods, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + }, + }, + { + Name: "Image is being updated too long in failed phase", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + }, + }, + Before: func(t *testing.T, deployment *Deployment) { + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.CreatePodName(testDeploymentName, k8sutil.ImageIDAndVersionRole, id, ""), + }, + Status: v1.PodStatus{ + Phase: v1.PodFailed, + }, + } + _, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).Create(&pod) + require.NoError(t, err) + }, + After: func(t *testing.T, deployment *Deployment) { + pods, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 0) + }, + }, + { + Name: "Image is being updated in not ready phase", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + }, + }, + RetrySoon: true, + Before: func(t *testing.T, deployment *Deployment) { + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.CreatePodName(testDeploymentName, k8sutil.ImageIDAndVersionRole, id, ""), + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.PodScheduled, + }, + }, + }, + } + _, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).Create(&pod) + require.NoError(t, err) + }, + After: func(t *testing.T, deployment *Deployment) { + pods, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + }, + }, + { + Name: "Image is being updated in ready phase with empty statuses list", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + }, + }, + RetrySoon: true, + Before: func(t *testing.T, deployment *Deployment) { + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.CreatePodName(testDeploymentName, k8sutil.ImageIDAndVersionRole, id, ""), + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + }, + } + _, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).Create(&pod) + require.NoError(t, err) + }, + After: func(t *testing.T, deployment *Deployment) { + pods, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + }, + }, + { + Name: "Can not get API version of arnagod", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testNewImage), + }, + }, + RetrySoon: true, + Before: func(t *testing.T, deployment *Deployment) { + pod := v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: k8sutil.CreatePodName(testDeploymentName, k8sutil.ImageIDAndVersionRole, id, ""), + }, + Status: v1.PodStatus{ + Conditions: []v1.PodCondition{ + { + Type: v1.PodReady, + Status: v1.ConditionTrue, + }, + }, + ContainerStatuses: []v1.ContainerStatus{ + {}, + }, + }, + } + _, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).Create(&pod) + require.NoError(t, err) + }, + After: func(t *testing.T, deployment *Deployment) { + pods, err := deployment.GetKubeCli().CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + }, + }, + } + + for _, testCase := range testCases { + //nolint:scopelint + t.Run(testCase.Name, func(t *testing.T) { + // Arrange + d, _ := createTestDeployment(Config{}, testCase.ArangoDeployment) + + d.status.last = api.DeploymentStatus{ + Images: createTestImages(false), + } + + if testCase.Before != nil { + testCase.Before(t, d) + } + + // Create custom resource in the fake kubernetes API + _, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(testNamespace).Create(d.apiObject) + require.NoError(t, err) + + // Act + retrySoon, err := d.ensureImages(d.apiObject) + + // Assert + assert.EqualValues(t, testCase.RetrySoon, retrySoon) + if testCase.ExpectedError != nil { + assert.EqualError(t, err, testCase.ExpectedError.Error()) + return + } + + require.NoError(t, err) + + if len(testCase.ExpectedPod.Spec.Containers) > 0 { + pods, err := d.deps.KubeCli.CoreV1().Pods(testNamespace).List(metav1.ListOptions{}) + require.NoError(t, err) + require.Len(t, pods.Items, 1) + require.Equal(t, testCase.ExpectedPod.Spec, pods.Items[0].Spec) + + ownerRef := pods.Items[0].GetOwnerReferences() + require.Len(t, ownerRef, 1) + require.Equal(t, ownerRef[0], testCase.ArangoDeployment.AsOwner()) + } + + if testCase.After != nil { + testCase.After(t, d) + } + }) + } +} + +func createTestCommandForImageUpdatePod() []string { + return []string{resources.ArangoDExecutor, + "--server.authentication=false", + fmt.Sprintf("--server.endpoint=tcp://[::]:%d", k8sutil.ArangoPort), + "--database.directory=" + k8sutil.ArangodVolumeMountDir, + "--log.output=+", + } +} + +func getTestTolerations() []v1.Toleration { + + shortDur := k8sutil.TolerationDuration{ + Forever: false, + TimeSpan: time.Second * 5, + } + + return []v1.Toleration{ + k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeNotReady, shortDur), + k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeUnreachable, shortDur), + k8sutil.NewNoExecuteToleration(k8sutil.TolerationKeyNodeAlphaUnreachable, shortDur), + } +} diff --git a/pkg/deployment/resources/certificates_tls.go b/pkg/deployment/resources/certificates_tls.go index 50dd325b2..263a3d94c 100644 --- a/pkg/deployment/resources/certificates_tls.go +++ b/pkg/deployment/resources/certificates_tls.go @@ -72,7 +72,9 @@ func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, // createTLSServerCertificate creates a TLS certificate for a specific server and stores // it in a secret with the given name. -func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, serverNames []string, spec api.TLSSpec, secretName string, ownerRef *metav1.OwnerReference) error { +func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, serverNames []string, spec api.TLSSpec, + secretName string, ownerRef *metav1.OwnerReference) error { + log = log.With().Str("secret", secretName).Logger() // Load alt names dnsNames, ipAddresses, emailAddress, err := spec.GetParsedAltNames() diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index ab45510b7..e24650bbb 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -43,6 +43,7 @@ import ( "github.com/pkg/errors" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" ) type optionPair struct { @@ -256,7 +257,8 @@ func createArangodArgs(apiObject metav1.Object, deplSpec api.DeploymentSpec, gro } // createArangoSyncArgs creates command line arguments for an arangosync server in the given group. -func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, agents api.MemberStatusList, id string) []string { +func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, + groupSpec api.ServerGroupSpec, id string) []string { options := make([]optionPair, 0, 64) var runCmd string var port int @@ -401,9 +403,6 @@ func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.Serve return nil, maskAny(err) } authorization = "bearer " + token - if err != nil { - return nil, maskAny(err) - } } else if group == api.ServerGroupSyncMasters { // Fall back to JWT secret secretData, err := r.getSyncJWTSecret(spec) @@ -477,8 +476,8 @@ func (r *Resources) createReadinessProbe(spec api.DeploymentSpec, group api.Serv return probeCfg, nil } -// createPodFinalizers creates a list of finalizers for a pod created for the given group. -func (r *Resources) createPodFinalizers(group api.ServerGroup) []string { +// CreatePodFinalizers creates a list of finalizers for a pod created for the given group. +func (r *Resources) CreatePodFinalizers(group api.ServerGroup) []string { switch group { case api.ServerGroupAgents: return []string{constants.FinalizerPodAgencyServing} @@ -489,8 +488,8 @@ func (r *Resources) createPodFinalizers(group api.ServerGroup) []string { } } -// createPodTolerations creates a list of tolerations for a pod created for the given group. -func (r *Resources) createPodTolerations(group api.ServerGroup, groupSpec api.ServerGroupSpec) []v1.Toleration { +// CreatePodTolerations creates a list of tolerations for a pod created for the given group. +func (r *Resources) CreatePodTolerations(group api.ServerGroup, groupSpec api.ServerGroupSpec) []v1.Toleration { notReadyDur := k8sutil.TolerationDuration{Forever: false, TimeSpan: time.Minute} unreachableDur := k8sutil.TolerationDuration{Forever: false, TimeSpan: time.Minute} switch group { @@ -548,17 +547,12 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, return maskAny(fmt.Errorf("Member '%s' not found", memberID)) } groupSpec := spec.GetServerGroupSpec(group) - lifecycleImage := r.context.GetLifecycleImage() - alpineImage := r.context.GetAlpineImage() - terminationGracePeriod := group.DefaultTerminationGracePeriod() - tolerations := r.createPodTolerations(group, groupSpec) - serviceAccountName := groupSpec.GetServiceAccountName() // Update pod name role := group.AsRole() roleAbbr := group.AsRoleAbbreviated() - podSuffix := createPodSuffix(spec) - m.PodName = k8sutil.CreatePodName(apiObject.GetName(), roleAbbr, m.ID, podSuffix) + + m.PodName = k8sutil.CreatePodName(apiObject.GetName(), roleAbbr, m.ID, CreatePodSuffix(spec)) newPhase := api.MemberPhaseCreated // Select image var imageInfo api.ImageInfo @@ -587,15 +581,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, newPhase = api.MemberPhaseUpgrading } args := createArangodArgs(apiObject, spec, group, status.Members.Agents, m.ID, version, autoUpgrade) - env := make(map[string]k8sutil.EnvValue) - livenessProbe, err := r.createLivenessProbe(spec, group) - if err != nil { - return maskAny(err) - } - readinessProbe, err := r.createReadinessProbe(spec, group, version) - if err != nil { - return maskAny(err) - } + tlsKeyfileSecretName := "" if spec.IsSecure() { tlsKeyfileSecretName = k8sutil.CreateTLSKeyfileSecretName(apiObject.GetName(), role, m.ID) @@ -626,19 +612,6 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, if err := k8sutil.ValidateTokenSecret(secrets, clusterJWTSecretName); err != nil { return maskAny(errors.Wrapf(err, "Cluster JWT secret validation failed")) } - } else { - env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{ - SecretName: spec.Authentication.GetJWTSecretName(), - SecretKey: constants.SecretKeyToken, - } - } - - } - - if spec.License.HasSecretName() { - env[constants.EnvArangoLicenseKey] = k8sutil.EnvValue{ - SecretName: spec.License.GetSecretName(), - SecretKey: constants.SecretKeyToken, } } @@ -659,12 +632,20 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, } } - engine := spec.GetStorageEngine().AsArangoArgument() - requireUUID := group == api.ServerGroupDBServers && m.IsInitialized - finalizers := r.createPodFinalizers(group) - if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, imageInfo.ImageID, lifecycleImage, alpineImage, spec.GetImagePullPolicy(), spec.ImagePullSecrets, - engine, requireUUID, terminationGracePeriod, args, env, finalizers, livenessProbe, readinessProbe, tolerations, serviceAccountName, tlsKeyfileSecretName, rocksdbEncryptionSecretName, - clusterJWTSecretName, groupSpec.GetNodeSelector(), groupSpec.PriorityClassName, groupSpec.Resources, exporter, groupSpec.GetSidecars(), groupSpec.VolumeClaimTemplate); err != nil { + memberPod := MemberArangoDPod{ + status: m, + tlsKeyfileSecretName: tlsKeyfileSecretName, + rocksdbEncryptionSecretName: rocksdbEncryptionSecretName, + clusterJWTSecretName: clusterJWTSecretName, + exporter: exporter, + groupSpec: groupSpec, + spec: spec, + group: group, + resources: r, + imageInfo: imageInfo, + } + + if err := CreateArangoPod(kubecli, apiObject, role, m.ID, m.PodName, args, &memberPod); err != nil { return maskAny(err) } @@ -733,31 +714,21 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, } // Prepare arguments - args := createArangoSyncArgs(apiObject, spec, group, groupSpec, status.Members.Agents, m.ID) - env := make(map[string]k8sutil.EnvValue) - if spec.Sync.Monitoring.GetTokenSecretName() != "" { - env[constants.EnvArangoSyncMonitoringToken] = k8sutil.EnvValue{ - SecretName: spec.Sync.Monitoring.GetTokenSecretName(), - SecretKey: constants.SecretKeyToken, - } - } - if spec.License.HasSecretName() { - env[constants.EnvArangoLicenseKey] = k8sutil.EnvValue{ - SecretName: spec.License.GetSecretName(), - SecretKey: constants.SecretKeyToken, - } - } - livenessProbe, err := r.createLivenessProbe(spec, group) - if err != nil { - return maskAny(err) - } - affinityWithRole := "" - if group == api.ServerGroupSyncWorkers { - affinityWithRole = api.ServerGroupDBServers.AsRole() + args := createArangoSyncArgs(apiObject, spec, group, groupSpec, m.ID) + + memberSyncPod := MemberSyncPod{ + tlsKeyfileSecretName: tlsKeyfileSecretName, + clientAuthCASecretName: clientAuthCASecretName, + masterJWTSecretName: masterJWTSecretName, + clusterJWTSecretName: clusterJWTSecretName, + groupSpec: groupSpec, + spec: spec, + group: group, + resources: r, + image: imageID, } - if err := k8sutil.CreateArangoSyncPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, imageID, lifecycleImage, spec.GetImagePullPolicy(), spec.ImagePullSecrets, terminationGracePeriod, args, env, - livenessProbe, tolerations, serviceAccountName, tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole, groupSpec.GetNodeSelector(), - groupSpec.PriorityClassName, groupSpec.Resources, groupSpec.GetSidecars()); err != nil { + + if err := CreateArangoPod(kubecli, apiObject, role, m.ID, m.PodName, args, &memberSyncPod); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") @@ -781,12 +752,49 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, return nil } +// CreateArangoPod creates a new Pod with container provided by parameter 'containerCreator' +// If the pod already exists, nil is returned. +// If another error occurs, that error is returned. +func CreateArangoPod(kubecli kubernetes.Interface, deployment k8sutil.APIObject, role, id, podName string, + args []string, podCreator k8sutil.PodCreator) error { + + // Prepare basic pod + p := k8sutil.NewPod(deployment.GetName(), role, id, podName, podCreator) + + podCreator.Init(&p) + + if initContainers, err := podCreator.GetInitContainers(); err != nil { + return maskAny(err) + } else if initContainers != nil { + p.Spec.InitContainers = append(p.Spec.InitContainers, initContainers...) + } + + c, err := k8sutil.NewContainer(args, podCreator.GetContainerCreator()) + if err != nil { + return maskAny(err) + } + + p.Spec.Volumes, c.VolumeMounts = podCreator.GetVolumes() + p.Spec.Containers = append(p.Spec.Containers, c) + podCreator.GetSidecars(&p) + + // Add (anti-)affinity + p.Spec.Affinity = k8sutil.CreateAffinity(deployment.GetName(), role, !podCreator.IsDeploymentMode(), + podCreator.GetAffinityRole()) + + if err := k8sutil.CreatePod(kubecli, &p, deployment.GetNamespace(), deployment.AsOwner()); err != nil { + return maskAny(err) + } + return nil +} + // EnsurePods creates all Pods listed in member status func (r *Resources) EnsurePods() error { iterator := r.context.GetServerGroupIterator() - status, _ := r.context.GetStatus() + deploymentStatus, _ := r.context.GetStatus() imageNotFoundOnce := &sync.Once{} - if err := iterator.ForeachServerGroup(func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error { + + createPodMember := func(group api.ServerGroup, groupSpec api.ServerGroupSpec, status *api.MemberStatusList) error { for _, m := range *status { if m.Phase != api.MemberPhaseNone { continue @@ -800,13 +808,16 @@ func (r *Resources) EnsurePods() error { } } return nil - }, &status); err != nil { + } + + if err := iterator.ForeachServerGroup(createPodMember, &deploymentStatus); err != nil { return maskAny(err) } + return nil } -func createPodSuffix(spec api.DeploymentSpec) string { +func CreatePodSuffix(spec api.DeploymentSpec) string { raw, _ := json.Marshal(spec) hash := sha1.Sum(raw) return fmt.Sprintf("%0x", hash)[:6] diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go new file mode 100644 index 000000000..b3f77b4db --- /dev/null +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -0,0 +1,261 @@ +package resources + +import ( + "math" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" +) + +const ( + ArangoDExecutor string = "/usr/sbin/arangod" +) + +type MemberArangoDPod struct { + status api.MemberStatus + tlsKeyfileSecretName string + rocksdbEncryptionSecretName string + clusterJWTSecretName string + exporter *k8sutil.ArangodbExporterContainerConf + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + context Context + resources *Resources + imageInfo api.ImageInfo +} + +type ArangoDContainer struct { + resources *Resources + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + imageInfo api.ImageInfo +} + +func (a *ArangoDContainer) GetExecutor() string { + return ArangoDExecutor +} + +func (a *ArangoDContainer) GetProbes() (*v1.Probe, *v1.Probe, error) { + var liveness, readiness *v1.Probe + + probeLivenessConfig, err := a.resources.createLivenessProbe(a.spec, a.group) + if err != nil { + return nil, nil, err + } + + probeReadinessConfig, err := a.resources.createReadinessProbe(a.spec, a.group, a.imageInfo.ArangoDBVersion) + if err != nil { + return nil, nil, err + } + + if probeLivenessConfig != nil { + liveness = probeLivenessConfig.Create() + } + + if probeReadinessConfig != nil { + readiness = probeReadinessConfig.Create() + } + + return liveness, readiness, nil +} + +func (a *ArangoDContainer) GetImage() string { + return a.imageInfo.ImageID +} + +func (a *ArangoDContainer) GetEnvs() []v1.EnvVar { + envs := make([]v1.EnvVar, 0) + + if a.spec.IsAuthenticated() { + if !versionHasJWTSecretKeyfile(a.imageInfo.ArangoDBVersion) { + env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangodJWTSecret, + a.spec.Authentication.GetJWTSecretName(), constants.SecretKeyToken) + + envs = append(envs, env) + } + } + + if a.spec.License.HasSecretName() { + env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, a.spec.License.GetSecretName(), + constants.SecretKeyToken) + + envs = append(envs, env) + } + + if a.resources.context.GetLifecycleImage() != "" { + envs = append(envs, k8sutil.GetLifecycleEnv()...) + } + + if len(envs) > 0 { + return envs + } + + return nil +} + +func (a *ArangoDContainer) GetResourceRequirements() v1.ResourceRequirements { + if a.groupSpec.GetVolumeClaimTemplate() != nil { + return a.groupSpec.Resources + } + + return k8sutil.ExtractPodResourceRequirement(a.groupSpec.Resources) +} + +func (a *ArangoDContainer) GetLifecycle() (*v1.Lifecycle, error) { + if a.resources.context.GetLifecycleImage() != "" { + return k8sutil.NewLifecycle() + } + return nil, nil +} + +func (a *ArangoDContainer) GetImagePullPolicy() v1.PullPolicy { + return a.spec.GetImagePullPolicy() +} + +func (m *MemberArangoDPod) Init(pod *v1.Pod) { + terminationGracePeriodSeconds := int64(math.Ceil(m.group.DefaultTerminationGracePeriod().Seconds())) + pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + pod.Spec.PriorityClassName = m.groupSpec.PriorityClassName +} + +func (m *MemberArangoDPod) GetImagePullSecrets() []string { + return m.spec.ImagePullSecrets +} + +func (m *MemberArangoDPod) GetAffinityRole() string { + return "" +} + +func (m *MemberArangoDPod) GetNodeSelector() map[string]string { + return m.groupSpec.GetNodeSelector() +} + +func (m *MemberArangoDPod) GetServiceAccountName() string { + return m.groupSpec.GetServiceAccountName() +} + +func (m *MemberArangoDPod) GetSidecars(pod *v1.Pod) { + if m.exporter != nil { + // Metrics sidecar + c := k8sutil.ArangodbexporterContainer(m.exporter.Image, m.exporter.Args, m.exporter.Env, m.exporter.LivenessProbe) + + if m.exporter.JWTTokenSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, k8sutil.ExporterJWTVolumeMount()) + } + + if m.tlsKeyfileSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) + } + + pod.Spec.Containers = append(pod.Spec.Containers, c) + pod.Labels[k8sutil.LabelKeyArangoExporter] = "yes" + } + + // A sidecar provided by the user + sidecars := m.groupSpec.GetSidecars() + if len(sidecars) > 0 { + pod.Spec.Containers = append(pod.Spec.Containers, sidecars...) + } + + return +} + +func (m *MemberArangoDPod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { + var volumes []v1.Volume + var volumeMounts []v1.VolumeMount + + volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount()) + + if m.resources.context.GetLifecycleImage() != "" { + volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMounts()) + } + + if m.status.PersistentVolumeClaimName != "" { + vol := k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName, + m.status.PersistentVolumeClaimName) + + volumes = append(volumes, vol) + } else { + volumes = append(volumes, k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) + } + + if m.tlsKeyfileSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, m.tlsKeyfileSecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount()) + } + + if m.rocksdbEncryptionSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.RocksdbEncryptionVolumeName, m.rocksdbEncryptionSecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.RocksdbEncryptionVolumeMount()) + } + + if m.exporter != nil && m.exporter.JWTTokenSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, m.exporter.JWTTokenSecretName) + volumes = append(volumes, vol) + } + + if m.clusterJWTSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, m.clusterJWTSecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount()) + } + + if m.resources.context.GetLifecycleImage() != "" { + volumes = append(volumes, k8sutil.LifecycleVolume()) + } + + return volumes, volumeMounts +} + +func (m *MemberArangoDPod) IsDeploymentMode() bool { + return m.spec.IsDevelopment() +} + +func (m *MemberArangoDPod) GetInitContainers() ([]v1.Container, error) { + var initContainers []v1.Container + + lifecycleImage := m.resources.context.GetLifecycleImage() + if lifecycleImage != "" { + c, err := k8sutil.InitLifecycleContainer(lifecycleImage) + if err != nil { + return nil, err + } + initContainers = append(initContainers, c) + } + + alpineImage := m.resources.context.GetAlpineImage() + if alpineImage != "" { + engine := m.spec.GetStorageEngine().AsArangoArgument() + requireUUID := m.group == api.ServerGroupDBServers && m.status.IsInitialized + + c := k8sutil.ArangodInitContainer("uuid", m.status.ID, engine, alpineImage, requireUUID) + initContainers = append(initContainers, c) + } + + return initContainers, nil +} + +func (m *MemberArangoDPod) GetFinalizers() []string { + return m.resources.CreatePodFinalizers(m.group) +} + +func (m *MemberArangoDPod) GetTolerations() []v1.Toleration { + return m.resources.CreatePodTolerations(m.group, m.groupSpec) +} + +func (m *MemberArangoDPod) GetContainerCreator() k8sutil.ContainerCreator { + return &ArangoDContainer{ + spec: m.spec, + group: m.group, + resources: m.resources, + imageInfo: m.imageInfo, + groupSpec: m.groupSpec, + } +} diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go new file mode 100644 index 000000000..891e8d1da --- /dev/null +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -0,0 +1,208 @@ +package resources + +import ( + "math" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" +) + +const ( + ArangoSyncExecutor string = "/usr/sbin/arangosync" +) + +type ArangoSyncContainer struct { + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + resources *Resources + image string +} + +type MemberSyncPod struct { + tlsKeyfileSecretName string + clientAuthCASecretName string + masterJWTSecretName string + clusterJWTSecretName string + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + resources *Resources + image string +} + +func (a *ArangoSyncContainer) GetExecutor() string { + return ArangoSyncExecutor +} + +func (a *ArangoSyncContainer) GetProbes() (*v1.Probe, *v1.Probe, error) { + livenessProbe, err := a.resources.createLivenessProbe(a.spec, a.group) + if err != nil { + return nil, nil, err + } + + if livenessProbe != nil { + return livenessProbe.Create(), nil, nil + } + + return nil, nil, nil +} + +func (a *ArangoSyncContainer) GetResourceRequirements() v1.ResourceRequirements { + return a.groupSpec.Resources +} + +func (a *ArangoSyncContainer) GetLifecycle() (*v1.Lifecycle, error) { + if a.resources.context.GetLifecycleImage() != "" { + return k8sutil.NewLifecycle() + } + return nil, nil +} + +func (a *ArangoSyncContainer) GetImagePullPolicy() v1.PullPolicy { + return a.spec.GetImagePullPolicy() +} + +func (a *ArangoSyncContainer) GetImage() string { + return a.image +} + +func (a *ArangoSyncContainer) GetEnvs() []v1.EnvVar { + envs := make([]v1.EnvVar, 0) + + if a.spec.Sync.Monitoring.GetTokenSecretName() != "" { + env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoSyncMonitoringToken, + a.spec.Sync.Monitoring.GetTokenSecretName(), constants.SecretKeyToken) + + envs = append(envs, env) + } + + if a.spec.License.HasSecretName() { + env := k8sutil.CreateEnvSecretKeySelector(constants.EnvArangoLicenseKey, a.spec.License.GetSecretName(), + constants.SecretKeyToken) + + envs = append(envs, env) + } + + if a.resources.context.GetLifecycleImage() != "" { + envs = append(envs, k8sutil.GetLifecycleEnv()...) + } + + if len(envs) > 0 { + return envs + } + + return nil +} + +func (m *MemberSyncPod) GetAffinityRole() string { + if m.group == api.ServerGroupSyncWorkers { + return api.ServerGroupDBServers.AsRole() + } + return "" +} + +func (m *MemberSyncPod) GetImagePullSecrets() []string { + return m.spec.ImagePullSecrets +} + +func (m *MemberSyncPod) GetNodeSelector() map[string]string { + return m.groupSpec.GetNodeSelector() +} + +func (m *MemberSyncPod) GetServiceAccountName() string { + return m.groupSpec.GetServiceAccountName() +} + +func (m *MemberSyncPod) GetSidecars(pod *v1.Pod) { + // A sidecar provided by the user + sidecars := m.groupSpec.GetSidecars() + if len(sidecars) > 0 { + pod.Spec.Containers = append(pod.Spec.Containers, sidecars...) + } +} + +func (m *MemberSyncPod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { + var volumes []v1.Volume + var volumeMounts []v1.VolumeMount + + if m.resources.context.GetLifecycleImage() != "" { + volumes = append(volumes, k8sutil.LifecycleVolume()) + volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMounts()) + } + + if m.tlsKeyfileSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, m.tlsKeyfileSecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount()) + } + + // Client Authentication certificate secret mount (if any) + if m.clientAuthCASecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClientAuthCAVolumeName, m.clientAuthCASecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.ClientAuthCACertificateVolumeMount()) + } + + // Master JWT secret mount (if any) + if m.masterJWTSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, m.masterJWTSecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.MasterJWTVolumeMount()) + } + + // Cluster JWT secret mount (if any) + if m.clusterJWTSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, m.clusterJWTSecretName) + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount()) + } + + return volumes, volumeMounts +} + +func (m *MemberSyncPod) IsDeploymentMode() bool { + return m.spec.IsDevelopment() +} + +func (m *MemberSyncPod) GetInitContainers() ([]v1.Container, error) { + var initContainers []v1.Container + + lifecycleImage := m.resources.context.GetLifecycleImage() + if lifecycleImage != "" { + c, err := k8sutil.InitLifecycleContainer(lifecycleImage) + if err != nil { + return nil, err + } + initContainers = append(initContainers, c) + } + + return initContainers, nil +} + +func (m *MemberSyncPod) GetFinalizers() []string { + return nil +} + +func (m *MemberSyncPod) GetTolerations() []v1.Toleration { + return m.resources.CreatePodTolerations(m.group, m.groupSpec) +} + +func (m *MemberSyncPod) GetContainerCreator() k8sutil.ContainerCreator { + return &ArangoSyncContainer{ + groupSpec: m.groupSpec, + spec: m.spec, + group: m.group, + resources: m.resources, + image: m.image, + } +} + +func (m *MemberSyncPod) Init(pod *v1.Pod) { + terminationGracePeriodSeconds := int64(math.Ceil(m.group.DefaultTerminationGracePeriod().Seconds())) + pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds + pod.Spec.PriorityClassName = m.groupSpec.PriorityClassName +} diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index fff0cc95f..56413d21f 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -285,7 +285,7 @@ func (r *Resources) GetExpectedPodArguments(apiObject metav1.Object, deplSpec ap } if group.IsArangosync() { groupSpec := deplSpec.GetServerGroupSpec(group) - return createArangoSyncArgs(apiObject, deplSpec, group, groupSpec, agents, id) + return createArangoSyncArgs(apiObject, deplSpec, group, groupSpec, id) } return nil } diff --git a/pkg/util/k8sutil/affinity.go b/pkg/util/k8sutil/affinity.go index f0f545ff7..8a3b1864b 100644 --- a/pkg/util/k8sutil/affinity.go +++ b/pkg/util/k8sutil/affinity.go @@ -27,10 +27,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// createAffinity creates pod anti-affinity for the given role. +// CreateAffinity creates pod anti-affinity for the given role. // role contains the name of the role to configure any-affinity with. // affinityWithRole contains the role to configure affinity with. -func createAffinity(deploymentName, role string, required bool, affinityWithRole string) *v1.Affinity { +func CreateAffinity(deploymentName, role string, required bool, affinityWithRole string) *v1.Affinity { a := &v1.Affinity{ NodeAffinity: &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ diff --git a/pkg/util/k8sutil/affinity_test.go b/pkg/util/k8sutil/affinity_test.go index 8841c0fc8..fa9c04663 100644 --- a/pkg/util/k8sutil/affinity_test.go +++ b/pkg/util/k8sutil/affinity_test.go @@ -31,7 +31,6 @@ import ( "github.com/stretchr/testify/require" ) -// TestCreateAffinity tests createAffinity func TestCreateAffinity(t *testing.T) { expectedNodeAffinity := &v1.NodeAffinity{ RequiredDuringSchedulingIgnoredDuringExecution: &v1.NodeSelector{ @@ -49,7 +48,7 @@ func TestCreateAffinity(t *testing.T) { }, } // Required - a := createAffinity("test", "role", true, "") + a := CreateAffinity("test", "role", true, "") assert.Nil(t, a.PodAffinity) require.NotNil(t, a.PodAntiAffinity) require.Len(t, a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 1) @@ -62,7 +61,7 @@ func TestCreateAffinity(t *testing.T) { assert.Equal(t, expectedNodeAffinity, a.NodeAffinity) // Require & affinity with role dbserver - a = createAffinity("test", "role", true, "dbserver") + a = CreateAffinity("test", "role", true, "dbserver") require.NotNil(t, a.PodAffinity) require.Len(t, a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 1) assert.Len(t, a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, 0) @@ -84,7 +83,7 @@ func TestCreateAffinity(t *testing.T) { assert.Equal(t, expectedNodeAffinity, a.NodeAffinity) // Not Required - a = createAffinity("test", "role", false, "") + a = CreateAffinity("test", "role", false, "") assert.Nil(t, a.PodAffinity) require.NotNil(t, a.PodAntiAffinity) assert.Len(t, a.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 0) @@ -97,7 +96,7 @@ func TestCreateAffinity(t *testing.T) { assert.Equal(t, expectedNodeAffinity, a.NodeAffinity) // Not Required & affinity with role dbserver - a = createAffinity("test", "role", false, "dbserver") + a = CreateAffinity("test", "role", false, "dbserver") require.NotNil(t, a.PodAffinity) require.Len(t, a.PodAffinity.PreferredDuringSchedulingIgnoredDuringExecution, 1) assert.Len(t, a.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution, 0) diff --git a/pkg/util/k8sutil/lifecycle.go b/pkg/util/k8sutil/lifecycle.go new file mode 100644 index 000000000..8e73ed8c3 --- /dev/null +++ b/pkg/util/k8sutil/lifecycle.go @@ -0,0 +1,80 @@ +package k8sutil + +import ( + "os" + "path/filepath" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + + v1 "k8s.io/api/core/v1" +) + +const ( + initLifecycleContainerName = "init-lifecycle" + lifecycleVolumeMountDir = "/lifecycle/tools" + lifecycleVolumeName = "lifecycle" +) + +// InitLifecycleContainer creates an init-container to copy the lifecycle binary to a shared volume. +func InitLifecycleContainer(image string) (v1.Container, error) { + binaryPath, err := os.Executable() + if err != nil { + return v1.Container{}, maskAny(err) + } + c := v1.Container{ + Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", lifecycleVolumeMountDir), + Name: initLifecycleContainerName, + Image: image, + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + LifecycleVolumeMounts(), + }, + SecurityContext: SecurityContextWithoutCapabilities(), + } + return c, nil +} + +// NewLifecycle creates a lifecycle structure with preStop handler. +func NewLifecycle() (*v1.Lifecycle, error) { + binaryPath, err := os.Executable() + if err != nil { + return nil, maskAny(err) + } + exePath := filepath.Join(lifecycleVolumeMountDir, filepath.Base(binaryPath)) + lifecycle := &v1.Lifecycle{ + PreStop: &v1.Handler{ + Exec: &v1.ExecAction{ + Command: append([]string{exePath}, "lifecycle", "preStop"), + }, + }, + } + + return lifecycle, nil +} + +func GetLifecycleEnv() []v1.EnvVar { + return []v1.EnvVar{ + CreateEnvFieldPath(constants.EnvOperatorPodName, "metadata.name"), + CreateEnvFieldPath(constants.EnvOperatorPodNamespace, "metadata.namespace"), + CreateEnvFieldPath(constants.EnvOperatorNodeName, "spec.nodeName"), + CreateEnvFieldPath(constants.EnvOperatorNodeNameArango, "spec.nodeName"), + } +} + +// LifecycleVolumeMounts creates a volume mount structure for shared lifecycle emptyDir. +func LifecycleVolumeMounts() v1.VolumeMount { + return v1.VolumeMount{ + Name: lifecycleVolumeName, + MountPath: lifecycleVolumeMountDir, + } +} + +// LifecycleVolume creates a volume mount structure for shared lifecycle emptyDir. +func LifecycleVolume() v1.Volume { + return v1.Volume{ + Name: lifecycleVolumeName, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + } +} diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index fe8f4824e..f75cd52c2 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -24,36 +24,28 @@ package k8sutil import ( "fmt" - "math" - "os" "path/filepath" "strings" "time" - "github.com/arangodb/kube-arangodb/pkg/util/constants" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" ) const ( - InitDataContainerName = "init-data" - InitLifecycleContainerName = "init-lifecycle" ServerContainerName = "server" ExporterContainerName = "exporter" - arangodVolumeName = "arangod-data" - tlsKeyfileVolumeName = "tls-keyfile" - lifecycleVolumeName = "lifecycle" - clientAuthCAVolumeName = "client-auth-ca" - clusterJWTSecretVolumeName = "cluster-jwt" - masterJWTSecretVolumeName = "master-jwt" - rocksdbEncryptionVolumeName = "rocksdb-encryption" - exporterJWTVolumeName = "exporter-jwt" + ArangodVolumeName = "arangod-data" + TlsKeyfileVolumeName = "tls-keyfile" + ClientAuthCAVolumeName = "client-auth-ca" + ClusterJWTSecretVolumeName = "cluster-jwt" + MasterJWTSecretVolumeName = "master-jwt" + RocksdbEncryptionVolumeName = "rocksdb-encryption" + ExporterJWTVolumeName = "exporter-jwt" ArangodVolumeMountDir = "/data" RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption" - JWTSecretFileVolumeMountDir = "/secrets/jwt" TLSKeyfileVolumeMountDir = "/secrets/tls" - LifecycleVolumeMountDir = "/lifecycle/tools" ClientAuthCAVolumeMountDir = "/secrets/client-auth/ca" ClusterJWTSecretVolumeMountDir = "/secrets/cluster/jwt" ExporterJWTVolumeMountDir = "/secrets/exporter/jwt" @@ -67,6 +59,31 @@ type EnvValue struct { SecretKey string // Key inside secret to fill into the envvar. Only relevant is SecretName is set. } +type PodCreator interface { + Init(*v1.Pod) + GetVolumes() ([]v1.Volume, []v1.VolumeMount) + GetSidecars(*v1.Pod) + GetInitContainers() ([]v1.Container, error) + GetFinalizers() []string + GetTolerations() []v1.Toleration + GetNodeSelector() map[string]string + GetServiceAccountName() string + GetAffinityRole() string + GetContainerCreator() ContainerCreator + GetImagePullSecrets() []string + IsDeploymentMode() bool +} + +type ContainerCreator interface { + GetExecutor() string + GetProbes() (*v1.Probe, *v1.Probe, error) + GetResourceRequirements() v1.ResourceRequirements + GetLifecycle() (*v1.Lifecycle, error) + GetImagePullPolicy() v1.PullPolicy + GetImage() string + GetEnvs() []v1.EnvVar +} + // CreateEnvVar creates an EnvVar structure for given key from given EnvValue. func (v EnvValue) CreateEnvVar(key string) v1.EnvVar { ev := v1.EnvVar{ @@ -75,6 +92,7 @@ func (v EnvValue) CreateEnvVar(key string) v1.EnvVar { if ev.Value != "" { ev.Value = v.Value } else if v.SecretName != "" { + //return CreateEnvSecretKeySelector(key, v.SecretName, v.SecretKey) ev.ValueFrom = &v1.EnvVarSource{ SecretKeyRef: &v1.SecretKeySelector{ LocalObjectReference: v1.LocalObjectReference{ @@ -199,82 +217,63 @@ func CreateTLSKeyfileSecretName(deploymentName, role, id string) string { return CreatePodName(deploymentName, role, id, "-tls-keyfile") } -// lifecycleVolumeMounts creates a volume mount structure for shared lifecycle emptyDir. -func lifecycleVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - {Name: lifecycleVolumeName, MountPath: LifecycleVolumeMountDir}, - } -} - -// arangodVolumeMounts creates a volume mount structure for arangod. -func arangodVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - {Name: arangodVolumeName, MountPath: ArangodVolumeMountDir}, +// ArangodVolumeMount creates a volume mount structure for arangod. +func ArangodVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: ArangodVolumeName, + MountPath: ArangodVolumeMountDir, } } -// tlsKeyfileVolumeMounts creates a volume mount structure for a TLS keyfile. -func tlsKeyfileVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - { - Name: tlsKeyfileVolumeName, - MountPath: TLSKeyfileVolumeMountDir, - }, +// TlsKeyfileVolumeMount creates a volume mount structure for a TLS keyfile. +func TlsKeyfileVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: TlsKeyfileVolumeName, + MountPath: TLSKeyfileVolumeMountDir, } } -// clientAuthCACertificateVolumeMounts creates a volume mount structure for a client-auth CA certificate (ca.crt). -func clientAuthCACertificateVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - { - Name: clientAuthCAVolumeName, - MountPath: ClientAuthCAVolumeMountDir, - }, +// ClientAuthCACertificateVolumeMount creates a volume mount structure for a client-auth CA certificate (ca.crt). +func ClientAuthCACertificateVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: ClientAuthCAVolumeName, + MountPath: ClientAuthCAVolumeMountDir, } } -// masterJWTVolumeMounts creates a volume mount structure for a master JWT secret (token). -func masterJWTVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - { - Name: masterJWTSecretVolumeName, - MountPath: MasterJWTSecretVolumeMountDir, - }, +// MasterJWTVolumeMount creates a volume mount structure for a master JWT secret (token). +func MasterJWTVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: MasterJWTSecretVolumeName, + MountPath: MasterJWTSecretVolumeMountDir, } } -// clusterJWTVolumeMounts creates a volume mount structure for a cluster JWT secret (token). -func clusterJWTVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - { - Name: clusterJWTSecretVolumeName, - MountPath: ClusterJWTSecretVolumeMountDir, - }, +// ClusterJWTVolumeMount creates a volume mount structure for a cluster JWT secret (token). +func ClusterJWTVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: ClusterJWTSecretVolumeName, + MountPath: ClusterJWTSecretVolumeMountDir, } } -func exporterJWTVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - { - Name: exporterJWTVolumeName, - MountPath: ExporterJWTVolumeMountDir, - }, +func ExporterJWTVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: ExporterJWTVolumeName, + MountPath: ExporterJWTVolumeMountDir, } } -// rocksdbEncryptionVolumeMounts creates a volume mount structure for a RocksDB encryption key. -func rocksdbEncryptionVolumeMounts() []v1.VolumeMount { - return []v1.VolumeMount{ - { - Name: rocksdbEncryptionVolumeName, - MountPath: RocksDBEncryptionVolumeMountDir, - }, +// RocksdbEncryptionVolumeMount creates a volume mount structure for a RocksDB encryption key. +func RocksdbEncryptionVolumeMount() v1.VolumeMount { + return v1.VolumeMount{ + Name: RocksdbEncryptionVolumeName, + MountPath: RocksDBEncryptionVolumeMountDir, } } -// arangodInitContainer creates a container configured to -// initalize a UUID file. -func arangodInitContainer(name, id, engine, alpineImage string, requireUUID bool) v1.Container { +// ArangodInitContainer creates a container configured to initalize a UUID file. +func ArangodInitContainer(name, id, engine, alpineImage string, requireUUID bool) v1.Container { uuidFile := filepath.Join(ArangodVolumeMountDir, "UUID") engineFile := filepath.Join(ArangodVolumeMountDir, "ENGINE") var command string @@ -297,9 +296,11 @@ func arangodInitContainer(name, id, engine, alpineImage string, requireUUID bool "-c", command, }, - Name: name, - Image: alpineImage, - VolumeMounts: arangodVolumeMounts(), + Name: name, + Image: alpineImage, + VolumeMounts: []v1.VolumeMount{ + ArangodVolumeMount(), + }, SecurityContext: SecurityContextWithoutCapabilities(), } return c @@ -325,57 +326,23 @@ func ExtractPodResourceRequirement(resources v1.ResourceRequirements) v1.Resourc } } -// arangodContainer creates a container configured to run `arangod`. -func arangodContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, - lifecycle *v1.Lifecycle, lifecycleEnvVars []v1.EnvVar, resources v1.ResourceRequirements, noFilterResources bool) v1.Container { - c := v1.Container{ - Command: append([]string{"/usr/sbin/arangod"}, args...), - Name: ServerContainerName, - Image: image, - ImagePullPolicy: imagePullPolicy, - Lifecycle: lifecycle, - Ports: []v1.ContainerPort{ - { - Name: "server", - ContainerPort: int32(ArangoPort), - Protocol: v1.ProtocolTCP, - }, - }, - VolumeMounts: arangodVolumeMounts(), - SecurityContext: SecurityContextWithoutCapabilities(), - } - if noFilterResources { - c.Resources = resources // if volumeclaimtemplate is specified - } else { - c.Resources = ExtractPodResourceRequirement(resources) // Storage is handled via pvcs - } +// NewContainer creates a container for specified creator +func NewContainer(args []string, containerCreator ContainerCreator) (v1.Container, error) { - for k, v := range env { - c.Env = append(c.Env, v.CreateEnvVar(k)) - } - if livenessProbe != nil { - c.LivenessProbe = livenessProbe.Create() - } - if readinessProbe != nil { - c.ReadinessProbe = readinessProbe.Create() - } - if lifecycle != nil { - c.Env = append(c.Env, lifecycleEnvVars...) - c.VolumeMounts = append(c.VolumeMounts, lifecycleVolumeMounts()...) + liveness, readiness, err := containerCreator.GetProbes() + if err != nil { + return v1.Container{}, err } - return c -} + lifecycle, err := containerCreator.GetLifecycle() + if err != nil { + return v1.Container{}, err + } -// arangosyncContainer creates a container configured to run `arangosync`. -func arangosyncContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, - lifecycle *v1.Lifecycle, lifecycleEnvVars []v1.EnvVar, resources v1.ResourceRequirements) v1.Container { - c := v1.Container{ - Command: append([]string{"/usr/sbin/arangosync"}, args...), - Name: ServerContainerName, - Image: image, - ImagePullPolicy: imagePullPolicy, - Lifecycle: lifecycle, + return v1.Container{ + Name: ServerContainerName, + Image: containerCreator.GetImage(), + Command: append([]string{containerCreator.GetExecutor()}, args...), Ports: []v1.ContainerPort{ { Name: "server", @@ -383,24 +350,17 @@ func arangosyncContainer(image string, imagePullPolicy v1.PullPolicy, args []str Protocol: v1.ProtocolTCP, }, }, - Resources: resources, + Env: containerCreator.GetEnvs(), + Resources: containerCreator.GetResourceRequirements(), + LivenessProbe: liveness, + ReadinessProbe: readiness, + Lifecycle: lifecycle, + ImagePullPolicy: containerCreator.GetImagePullPolicy(), SecurityContext: SecurityContextWithoutCapabilities(), - } - for k, v := range env { - c.Env = append(c.Env, v.CreateEnvVar(k)) - } - if livenessProbe != nil { - c.LivenessProbe = livenessProbe.Create() - } - if lifecycle != nil { - c.Env = append(c.Env, lifecycleEnvVars...) - c.VolumeMounts = append(c.VolumeMounts, lifecycleVolumeMounts()...) - } - - return c + }, nil } -func arangodbexporterContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { +func ArangodbexporterContainer(image string, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { c := v1.Container{ Command: append([]string{"/app/arangodb-exporter"}, args...), Name: ExporterContainerName, @@ -424,103 +384,28 @@ func arangodbexporterContainer(image string, imagePullPolicy v1.PullPolicy, args return c } -// newLifecycle creates a lifecycle structure with preStop handler. -func newLifecycle() (*v1.Lifecycle, []v1.EnvVar, []v1.Volume, error) { - binaryPath, err := os.Executable() - if err != nil { - return nil, nil, nil, maskAny(err) - } - exePath := filepath.Join(LifecycleVolumeMountDir, filepath.Base(binaryPath)) - lifecycle := &v1.Lifecycle{ - PreStop: &v1.Handler{ - Exec: &v1.ExecAction{ - Command: append([]string{exePath}, "lifecycle", "preStop"), - }, - }, - } - envVars := []v1.EnvVar{ - v1.EnvVar{ - Name: constants.EnvOperatorPodName, - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.name", - }, - }, - }, - v1.EnvVar{ - Name: constants.EnvOperatorPodNamespace, - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "metadata.namespace", - }, - }, - }, - v1.EnvVar{ - Name: constants.EnvOperatorNodeName, - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - v1.EnvVar{ - Name: constants.EnvOperatorNodeNameArango, - ValueFrom: &v1.EnvVarSource{ - FieldRef: &v1.ObjectFieldSelector{ - FieldPath: "spec.nodeName", - }, - }, - }, - } - vols := []v1.Volume{ - v1.Volume{ - Name: lifecycleVolumeName, - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - }, - } - return lifecycle, envVars, vols, nil -} +// NewPod creates a basic Pod for given settings. +func NewPod(deploymentName, role, id, podName string, podCreator PodCreator) v1.Pod { -// initLifecycleContainer creates an init-container to copy the lifecycle binary -// to a shared volume. -func initLifecycleContainer(image string) (v1.Container, error) { - binaryPath, err := os.Executable() - if err != nil { - return v1.Container{}, maskAny(err) - } - c := v1.Container{ - Command: append([]string{binaryPath}, "lifecycle", "copy", "--target", LifecycleVolumeMountDir), - Name: InitLifecycleContainerName, - Image: image, - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: lifecycleVolumeMounts(), - SecurityContext: SecurityContextWithoutCapabilities(), - } - return c, nil -} - -// newPod creates a basic Pod for given settings. -func newPod(deploymentName, ns, role, id, podName string, imagePullSecrets []string, finalizers []string, tolerations []v1.Toleration, serviceAccountName string, nodeSelector map[string]string) v1.Pod { hostname := CreatePodHostName(deploymentName, role, id) p := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name: podName, Labels: LabelsForDeployment(deploymentName, role), - Finalizers: finalizers, + Finalizers: podCreator.GetFinalizers(), }, Spec: v1.PodSpec{ Hostname: hostname, Subdomain: CreateHeadlessServiceName(deploymentName), RestartPolicy: v1.RestartPolicyNever, - Tolerations: tolerations, - ServiceAccountName: serviceAccountName, - NodeSelector: nodeSelector, + Tolerations: podCreator.GetTolerations(), + ServiceAccountName: podCreator.GetServiceAccountName(), + NodeSelector: podCreator.GetNodeSelector(), }, } // Add ImagePullSecrets + imagePullSecrets := podCreator.GetImagePullSecrets() if imagePullSecrets != nil { imagePullSecretsReference := make([]v1.LocalObjectReference, len(imagePullSecrets)) for id := range imagePullSecrets { @@ -543,299 +428,78 @@ type ArangodbExporterContainerConf struct { Image string } -// CreateArangodPod creates a Pod that runs `arangod`. +// CreatePod adds an owner to the given pod and calls the k8s api-server to created it. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, - role, id, podName, pvcName, image, lifecycleImage, alpineImage string, - imagePullPolicy v1.PullPolicy, imagePullSecrets []string, - engine string, requireUUID bool, terminationGracePeriod time.Duration, - args []string, env map[string]EnvValue, finalizers []string, - livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, tolerations []v1.Toleration, serviceAccountName string, - tlsKeyfileSecretName, rocksdbEncryptionSecretName string, clusterJWTSecretName string, nodeSelector map[string]string, - podPriorityClassName string, resources v1.ResourceRequirements, exporter *ArangodbExporterContainerConf, sidecars []v1.Container, vct *v1.PersistentVolumeClaim) error { - - // Prepare basic pod - p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, imagePullSecrets, finalizers, tolerations, serviceAccountName, nodeSelector) - terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) - p.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds - - // Add lifecycle container - var lifecycle *v1.Lifecycle - var lifecycleEnvVars []v1.EnvVar - var lifecycleVolumes []v1.Volume - if lifecycleImage != "" { - c, err := initLifecycleContainer(lifecycleImage) - if err != nil { - return maskAny(err) - } - p.Spec.InitContainers = append(p.Spec.InitContainers, c) - lifecycle, lifecycleEnvVars, lifecycleVolumes, err = newLifecycle() - if err != nil { - return maskAny(err) - } - } - - // Add arangod container - c := - arangodContainer(image, imagePullPolicy, args, env, livenessProbe, readinessProbe, lifecycle, lifecycleEnvVars, resources, vct != nil) - if tlsKeyfileSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) - } - if rocksdbEncryptionSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, rocksdbEncryptionVolumeMounts()...) - } - if clusterJWTSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, clusterJWTVolumeMounts()...) - } - - p.Spec.Containers = append(p.Spec.Containers, c) - - // Add arangodb exporter container - if exporter != nil { - c = arangodbexporterContainer(exporter.Image, imagePullPolicy, exporter.Args, exporter.Env, exporter.LivenessProbe) - if exporter.JWTTokenSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, exporterJWTVolumeMounts()...) - } - if tlsKeyfileSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) - } - p.Spec.Containers = append(p.Spec.Containers, c) - p.Labels[LabelKeyArangoExporter] = "yes" - } - - // Add sidecars - if len(sidecars) > 0 { - p.Spec.Containers = append(p.Spec.Containers, sidecars...) - } - - // Add priorityClassName - p.Spec.PriorityClassName = podPriorityClassName - - // Add UUID init container - if alpineImage != "" { - p.Spec.InitContainers = append(p.Spec.InitContainers, arangodInitContainer("uuid", id, engine, alpineImage, requireUUID)) - } - - // Add volume - if pvcName != "" { - // Create PVC - vol := v1.Volume{ - Name: arangodVolumeName, - VolumeSource: v1.VolumeSource{ - PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ - ClaimName: pvcName, - }, - }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } else { - // Create emptydir volume - vol := v1.Volume{ - Name: arangodVolumeName, - VolumeSource: v1.VolumeSource{ - EmptyDir: &v1.EmptyDirVolumeSource{}, - }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } - - // TLS keyfile secret mount (if any) - if tlsKeyfileSecretName != "" { - vol := v1.Volume{ - Name: tlsKeyfileVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: tlsKeyfileSecretName, - }, - }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } - - // RocksDB encryption secret mount (if any) - if rocksdbEncryptionSecretName != "" { - vol := v1.Volume{ - Name: rocksdbEncryptionVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: rocksdbEncryptionSecretName, - }, - }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } - - // Exporter Token Mount - if exporter != nil && exporter.JWTTokenSecretName != "" { - vol := v1.Volume{ - Name: exporterJWTVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: exporter.JWTTokenSecretName, - }, - }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } - - // Cluster JWT secret mount (if any) - if clusterJWTSecretName != "" { - vol := v1.Volume{ - Name: clusterJWTSecretVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: clusterJWTSecretName, - }, - }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } - - // Lifecycle volumes (if any) - p.Spec.Volumes = append(p.Spec.Volumes, lifecycleVolumes...) - - // Add (anti-)affinity - p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, "") - - if err := createPod(kubecli, &p, deployment.GetNamespace(), deployment.AsOwner()); err != nil { +func CreatePod(kubecli kubernetes.Interface, pod *v1.Pod, ns string, owner metav1.OwnerReference) error { + addOwnerRefToObject(pod.GetObjectMeta(), &owner) + if _, err := kubecli.CoreV1().Pods(ns).Create(pod); err != nil && !IsAlreadyExists(err) { return maskAny(err) } return nil } -// CreateArangoSyncPod creates a Pod that runs `arangosync`. -// If the pod already exists, nil is returned. -// If another error occurs, that error is returned. -func CreateArangoSyncPod(kubecli kubernetes.Interface, developmentMode bool, deployment APIObject, role, id, podName, image, lifecycleImage string, - imagePullPolicy v1.PullPolicy, imagePullSecrets []string, - terminationGracePeriod time.Duration, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig, tolerations []v1.Toleration, serviceAccountName string, - tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName, affinityWithRole string, nodeSelector map[string]string, - podPriorityClassName string, resources v1.ResourceRequirements, sidecars []v1.Container) error { - // Prepare basic pod - p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, imagePullSecrets, nil, tolerations, serviceAccountName, nodeSelector) - terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) - p.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds - - // Add lifecycle container - var lifecycle *v1.Lifecycle - var lifecycleEnvVars []v1.EnvVar - var lifecycleVolumes []v1.Volume - if lifecycleImage != "" { - c, err := initLifecycleContainer(lifecycleImage) - if err != nil { - return maskAny(err) - } - p.Spec.InitContainers = append(p.Spec.InitContainers, c) - lifecycle, lifecycleEnvVars, lifecycleVolumes, err = newLifecycle() - if err != nil { - return maskAny(err) - } - } - - // Lifecycle volumes (if any) - p.Spec.Volumes = append(p.Spec.Volumes, lifecycleVolumes...) - - // Add arangosync container - c := arangosyncContainer(image, imagePullPolicy, args, env, livenessProbe, lifecycle, lifecycleEnvVars, resources) - if tlsKeyfileSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) - } - if clientAuthCASecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, clientAuthCACertificateVolumeMounts()...) - } - if masterJWTSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, masterJWTVolumeMounts()...) - } - if clusterJWTSecretName != "" { - c.VolumeMounts = append(c.VolumeMounts, clusterJWTVolumeMounts()...) +func SecurityContextWithoutCapabilities() *v1.SecurityContext { + return &v1.SecurityContext{ + Capabilities: &v1.Capabilities{ + Drop: []v1.Capability{"ALL"}, + }, } - p.Spec.Containers = append(p.Spec.Containers, c) +} - // Add sidecars - if len(sidecars) > 0 { - p.Spec.Containers = append(p.Spec.Containers, sidecars...) +func CreateVolumeEmptyDir(name string) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, } +} - // Add priorityClassName - p.Spec.PriorityClassName = podPriorityClassName - - // TLS keyfile secret mount (if any) - if tlsKeyfileSecretName != "" { - vol := v1.Volume{ - Name: tlsKeyfileVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: tlsKeyfileSecretName, - }, +func CreateVolumeWithSecret(name, secretName string) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: secretName, }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) + }, } +} - // Client Authentication certificate secret mount (if any) - if clientAuthCASecretName != "" { - vol := v1.Volume{ - Name: clientAuthCAVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: clientAuthCASecretName, - }, +func CreateVolumeWithPersitantVolumeClaim(name, claimName string) v1.Volume { + return v1.Volume{ + Name: name, + VolumeSource: v1.VolumeSource{ + PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{ + ClaimName: claimName, }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) + }, } +} - // Master JWT secret mount (if any) - if masterJWTSecretName != "" { - vol := v1.Volume{ - Name: masterJWTSecretVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: masterJWTSecretName, - }, +func CreateEnvFieldPath(name, fieldPath string) v1.EnvVar { + return v1.EnvVar{ + Name: name, + ValueFrom: &v1.EnvVarSource{ + FieldRef: &v1.ObjectFieldSelector{ + FieldPath: fieldPath, }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) + }, } +} - // Cluster JWT secret mount (if any) - if clusterJWTSecretName != "" { - vol := v1.Volume{ - Name: clusterJWTSecretVolumeName, - VolumeSource: v1.VolumeSource{ - Secret: &v1.SecretVolumeSource{ - SecretName: clusterJWTSecretName, +func CreateEnvSecretKeySelector(name, SecretKeyName, secretKey string) v1.EnvVar { + return v1.EnvVar{ + Name: name, + Value: "", + ValueFrom: &v1.EnvVarSource{ + SecretKeyRef: &v1.SecretKeySelector{ + LocalObjectReference: v1.LocalObjectReference{ + Name: SecretKeyName, }, + Key: secretKey, }, - } - p.Spec.Volumes = append(p.Spec.Volumes, vol) - } - - // Add (anti-)affinity - p.Spec.Affinity = createAffinity(deployment.GetName(), role, !developmentMode, affinityWithRole) - - if err := createPod(kubecli, &p, deployment.GetNamespace(), deployment.AsOwner()); err != nil { - return maskAny(err) - } - return nil -} - -// createPod adds an owner to the given pod and calls the k8s api-server to created it. -// If the pod already exists, nil is returned. -// If another error occurs, that error is returned. -func createPod(kubecli kubernetes.Interface, pod *v1.Pod, ns string, owner metav1.OwnerReference) error { - addOwnerRefToObject(pod.GetObjectMeta(), &owner) - if _, err := kubecli.CoreV1().Pods(ns).Create(pod); err != nil && !IsAlreadyExists(err) { - return maskAny(err) - } - return nil -} - -func SecurityContextWithoutCapabilities() *v1.SecurityContext { - return &v1.SecurityContext{ - Capabilities: &v1.Capabilities{ - Drop: []v1.Capability{"ALL"}, }, } } diff --git a/reboot.go b/reboot.go index 4d5266a82..40f42fc1b 100644 --- a/reboot.go +++ b/reboot.go @@ -177,14 +177,7 @@ func runVolumeInspector(ctx context.Context, kube kubernetes.Interface, ns, name }, }, Volumes: []corev1.Volume{ - corev1.Volume{ - Name: "data", - VolumeSource: corev1.VolumeSource{ - PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ - ClaimName: claimname, - }, - }, - }, + k8sutil.CreateVolumeWithPersitantVolumeClaim("data", claimname), }, }, } From 45e05443e43c21b383ec0b4eb8740b377eaeacf0 Mon Sep 17 00:00:00 2001 From: informalict Date: Fri, 22 Nov 2019 09:11:36 +0100 Subject: [PATCH 2/3] Clean naming functions --- pkg/deployment/deployment_test.go | 12 ++++++------ pkg/deployment/resources/pod_creator_arangod.go | 2 +- pkg/deployment/resources/pod_creator_sync.go | 2 +- pkg/util/k8sutil/lifecycle.go | 6 +++--- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pkg/deployment/deployment_test.go b/pkg/deployment/deployment_test.go index 8bec168ba..7fa254da1 100644 --- a/pkg/deployment/deployment_test.go +++ b/pkg/deployment/deployment_test.go @@ -1081,7 +1081,7 @@ func TestEnsurePods(t *testing.T) { Ports: createTestPorts(), VolumeMounts: []v1.VolumeMount{ k8sutil.ArangodVolumeMount(), - k8sutil.LifecycleVolumeMounts(), + k8sutil.LifecycleVolumeMount(), }, Lifecycle: createTestLifecycle(), LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), @@ -1167,7 +1167,7 @@ func TestEnsurePods(t *testing.T) { Ports: createTestPorts(), VolumeMounts: []v1.VolumeMount{ k8sutil.ArangodVolumeMount(), - k8sutil.LifecycleVolumeMounts(), + k8sutil.LifecycleVolumeMount(), }, Lifecycle: createTestLifecycle(), LivenessProbe: createTestLivenessProbe(false, "", k8sutil.ArangoPort), @@ -1277,7 +1277,7 @@ func TestEnsurePods(t *testing.T) { }, VolumeMounts: []v1.VolumeMount{ k8sutil.ArangodVolumeMount(), - k8sutil.LifecycleVolumeMounts(), + k8sutil.LifecycleVolumeMount(), k8sutil.TlsKeyfileVolumeMount(), k8sutil.RocksdbEncryptionVolumeMount(), k8sutil.ClusterJWTVolumeMount(), @@ -1749,7 +1749,7 @@ func TestEnsurePods(t *testing.T) { Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, VolumeMounts: []v1.VolumeMount{ - k8sutil.LifecycleVolumeMounts(), + k8sutil.LifecycleVolumeMount(), k8sutil.TlsKeyfileVolumeMount(), k8sutil.ClientAuthCACertificateVolumeMount(), k8sutil.MasterJWTVolumeMount(), @@ -1846,7 +1846,7 @@ func TestEnsurePods(t *testing.T) { Capabilities: &v1.Capabilities{Drop: []v1.Capability{"ALL"}}, }, VolumeMounts: []v1.VolumeMount{ - k8sutil.LifecycleVolumeMounts(), + k8sutil.LifecycleVolumeMount(), k8sutil.MasterJWTVolumeMount(), }, }, @@ -2293,7 +2293,7 @@ func createTestLifecycleContainer() v1.Container { Image: testImageLifecycle, Command: []string{binaryPath, "lifecycle", "copy", "--target", "/lifecycle/tools"}, VolumeMounts: []v1.VolumeMount{ - k8sutil.LifecycleVolumeMounts(), + k8sutil.LifecycleVolumeMount(), }, ImagePullPolicy: "IfNotPresent", SecurityContext: &v1.SecurityContext{ diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index b3f77b4db..2d6070fe6 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -172,7 +172,7 @@ func (m *MemberArangoDPod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount()) if m.resources.context.GetLifecycleImage() != "" { - volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMounts()) + volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount()) } if m.status.PersistentVolumeClaimName != "" { diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 891e8d1da..55b34702f 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -131,7 +131,7 @@ func (m *MemberSyncPod) GetVolumes() ([]v1.Volume, []v1.VolumeMount) { if m.resources.context.GetLifecycleImage() != "" { volumes = append(volumes, k8sutil.LifecycleVolume()) - volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMounts()) + volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount()) } if m.tlsKeyfileSecretName != "" { diff --git a/pkg/util/k8sutil/lifecycle.go b/pkg/util/k8sutil/lifecycle.go index 8e73ed8c3..b134cfa91 100644 --- a/pkg/util/k8sutil/lifecycle.go +++ b/pkg/util/k8sutil/lifecycle.go @@ -27,7 +27,7 @@ func InitLifecycleContainer(image string) (v1.Container, error) { Image: image, ImagePullPolicy: v1.PullIfNotPresent, VolumeMounts: []v1.VolumeMount{ - LifecycleVolumeMounts(), + LifecycleVolumeMount(), }, SecurityContext: SecurityContextWithoutCapabilities(), } @@ -61,8 +61,8 @@ func GetLifecycleEnv() []v1.EnvVar { } } -// LifecycleVolumeMounts creates a volume mount structure for shared lifecycle emptyDir. -func LifecycleVolumeMounts() v1.VolumeMount { +// LifecycleVolumeMount creates a volume mount structure for shared lifecycle emptyDir. +func LifecycleVolumeMount() v1.VolumeMount { return v1.VolumeMount{ Name: lifecycleVolumeName, MountPath: lifecycleVolumeMountDir, From 9aec33ce9e5775787eed2d58f148b65b7c19b990 Mon Sep 17 00:00:00 2001 From: informalict Date: Mon, 25 Nov 2019 07:09:11 +0100 Subject: [PATCH 3/3] Add disclaimers to the new files --- pkg/deployment/deployment_test.go | 22 +++++++++++++++++++ pkg/deployment/images_test.go | 22 +++++++++++++++++++ .../resources/pod_creator_arangod.go | 22 +++++++++++++++++++ pkg/deployment/resources/pod_creator_sync.go | 22 +++++++++++++++++++ pkg/util/k8sutil/lifecycle.go | 22 +++++++++++++++++++ 5 files changed, 110 insertions(+) diff --git a/pkg/deployment/deployment_test.go b/pkg/deployment/deployment_test.go index 7fa254da1..e2676a9a3 100644 --- a/pkg/deployment/deployment_test.go +++ b/pkg/deployment/deployment_test.go @@ -1,3 +1,25 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + package deployment import ( diff --git a/pkg/deployment/images_test.go b/pkg/deployment/images_test.go index 6cc02a2d6..5bb769289 100644 --- a/pkg/deployment/images_test.go +++ b/pkg/deployment/images_test.go @@ -1,3 +1,25 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + package deployment import ( diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index 2d6070fe6..e81f3aedf 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -1,3 +1,25 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + package resources import ( diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 55b34702f..8dea1fd7e 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -1,3 +1,25 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + package resources import ( diff --git a/pkg/util/k8sutil/lifecycle.go b/pkg/util/k8sutil/lifecycle.go index b134cfa91..1e86a8ebb 100644 --- a/pkg/util/k8sutil/lifecycle.go +++ b/pkg/util/k8sutil/lifecycle.go @@ -1,3 +1,25 @@ +// +// DISCLAIMER +// +// Copyright 2019 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Tomasz Mielech +// + package k8sutil import (