From 3846f0411380499d85951e650bbdfd9d09c55f07 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Sun, 7 Nov 2021 18:23:14 +0100 Subject: [PATCH 1/9] add GetArgs and GetName to the ContainerCreator interface --- pkg/deployment/images.go | 12 +++++++++++- pkg/deployment/resources/pod_creator.go | 14 ++++++-------- .../resources/pod_creator_arangod.go | 19 +++++++++++++++---- pkg/deployment/resources/pod_creator_sync.go | 11 +++++++++++ pkg/util/k8sutil/interfaces/pod_creator.go | 2 ++ pkg/util/k8sutil/pods.go | 6 +++--- 6 files changed, 48 insertions(+), 16 deletions(-) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 40f4e01a9..a4a1e28aa 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -54,6 +54,7 @@ type ImageUpdatePod struct { // ContainerIdentity helps to resolve the container identity, e.g.: image ID, version of the entrypoint. type ContainerIdentity struct { + args []string ID *api.ServerIDGroupSpec image string imagePullPolicy core.PullPolicy @@ -222,6 +223,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, cac apiObject: ib.APIObject, containerCreator: &ArangoDIdentity{ ContainerCreator: &ContainerIdentity{ + args: args, ID: ib.Spec.ID, image: image, imagePullPolicy: ib.Spec.GetImagePullPolicy(), @@ -230,7 +232,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, cac }, } - pod, err = resources.RenderArangoPod(cachedStatus, ib.APIObject, role, id, podName, args, &imagePod) + pod, err = resources.RenderArangoPod(cachedStatus, ib.APIObject, role, id, podName, &imagePod) if err != nil { log.Debug().Err(err).Msg("Failed to render image ID pod") return true, errors.WithStack(err) @@ -376,6 +378,10 @@ func (i *ImageUpdatePod) ApplyPodSpec(_ *core.PodSpec) error { return nil } +func (a *ContainerIdentity) GetArgs() []string { + return a.args +} + func (a *ContainerIdentity) GetEnvs() []core.EnvVar { return nil } @@ -396,6 +402,10 @@ func (a *ContainerIdentity) GetLifecycle() (*core.Lifecycle, error) { return nil, nil } +func (a *ContainerIdentity) GetName() string { + return k8sutil.ServerContainerName +} + func (a *ContainerIdentity) GetPorts() []core.ContainerPort { return []core.ContainerPort{ { diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 8d526fca6..a3352cdbe 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -343,7 +343,6 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect newMember.PodName = k8sutil.CreatePodName(apiObject.GetName(), roleAbbr, newMember.ID, CreatePodSuffix(spec)) var podCreator interfaces.PodCreator - var args []string if group.IsArangod() { // Prepare arguments autoUpgrade := newMember.Conditions.IsTrue(api.ConditionTypeAutoUpgrade) || spec.Upgrade.Get().AutoUpgrade @@ -365,7 +364,7 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect input := memberPod.AsInput() var err error - args, err = createArangodArgs(cachedStatus, input) + memberPod.args, err = createArangodArgs(cachedStatus, input) if err != nil { return nil, errors.WithStack(err) } @@ -428,9 +427,7 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect } } - // Prepare arguments - args = createArangoSyncArgs(apiObject, spec, group, groupSpec, *newMember) - + // Prepare arguments. memberSyncPod := MemberSyncPod{ tlsKeyfileSecretName: tlsKeyfileSecretName, clientAuthCASecretName: clientAuthCASecretName, @@ -442,6 +439,7 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect resources: r, imageInfo: imageInfo, arangoMember: *member, + args: createArangoSyncArgs(apiObject, spec, group, groupSpec, *newMember), } podCreator = &memberSyncPod @@ -449,7 +447,7 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect return nil, errors.Newf("unable to render Pod") } - pod, err := RenderArangoPod(cachedStatus, apiObject, role, newMember.ID, newMember.PodName, args, podCreator) + pod, err := RenderArangoPod(cachedStatus, apiObject, role, newMember.ID, newMember.PodName, podCreator) if err != nil { return nil, err } @@ -637,7 +635,7 @@ func (r *Resources) createPodForMember(ctx context.Context, cachedStatus inspect // RenderArangoPod renders new ArangoD Pod func RenderArangoPod(cachedStatus inspectorInterface.Inspector, deployment k8sutil.APIObject, role, id, podName string, - args []string, podCreator interfaces.PodCreator) (*core.Pod, error) { + podCreator interfaces.PodCreator) (*core.Pod, error) { // Prepare basic pod p := k8sutil.NewPod(deployment.GetName(), role, id, podName, podCreator) @@ -666,7 +664,7 @@ func RenderArangoPod(cachedStatus inspectorInterface.Inspector, deployment k8sut p.Spec.InitContainers = append(p.Spec.InitContainers, initContainers...) } - c, err := k8sutil.NewContainer(args, podCreator.GetContainerCreator()) + c, err := k8sutil.NewContainer(podCreator.GetContainerCreator()) if err != nil { return nil, errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index a075c2325..cd831cb15 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -67,6 +67,7 @@ type MemberArangoDPod struct { imageInfo api.ImageInfo autoUpgrade bool id string + args []string } type ArangoDContainer struct { @@ -76,6 +77,7 @@ type ArangoDContainer struct { spec api.DeploymentSpec group api.ServerGroup imageInfo api.ImageInfo + args []string } func (a *ArangoDContainer) GetPorts() []core.ContainerPort { @@ -101,6 +103,14 @@ func (a *ArangoDContainer) GetPorts() []core.ContainerPort { return ports } +func (a *ArangoDContainer) GetArgs() []string { + return a.args +} + +func (a *ArangoDContainer) GetName() string { + return k8sutil.ServerContainerName +} + func (a *ArangoDContainer) GetExecutor() string { return a.groupSpec.GetEntrypoint(ArangoDExecutor) } @@ -414,12 +424,12 @@ func (m *MemberArangoDPod) GetInitContainers(cachedStatus interfaces.Inspector) { // Upgrade container - run in background if m.autoUpgrade || m.status.Upgrade { - args, err := createArangodArgsWithUpgrade(cachedStatus, m.AsInput()) + m.args, err = createArangodArgsWithUpgrade(cachedStatus, m.AsInput()) if err != nil { return nil, err } - c, err := k8sutil.NewContainer(args, m.GetContainerCreator()) + c, err := k8sutil.NewContainer(m.GetContainerCreator()) if err != nil { return nil, err } @@ -438,12 +448,12 @@ func (m *MemberArangoDPod) GetInitContainers(cachedStatus interfaces.Inspector) { versionArgs := pod.UpgradeVersionCheck().Args(m.AsInput()) if len(versionArgs) > 0 { - args, err := createArangodArgs(cachedStatus, m.AsInput(), versionArgs...) + m.args, err = createArangodArgs(cachedStatus, m.AsInput(), versionArgs...) if err != nil { return nil, err } - c, err := k8sutil.NewContainer(args, m.GetContainerCreator()) + c, err := k8sutil.NewContainer(m.GetContainerCreator()) if err != nil { return nil, err } @@ -496,6 +506,7 @@ func (m *MemberArangoDPod) GetContainerCreator() interfaces.ContainerCreator { resources: m.resources, imageInfo: m.imageInfo, groupSpec: m.groupSpec, + args: m.args, } } diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index f1b8d3cb8..d9477c0dc 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -48,6 +48,7 @@ type ArangoSyncContainer struct { group api.ServerGroup resources *Resources imageInfo api.ImageInfo + args []string } var _ interfaces.PodCreator = &MemberSyncPod{} @@ -64,6 +65,15 @@ type MemberSyncPod struct { arangoMember api.ArangoMember resources *Resources imageInfo api.ImageInfo + args []string +} + +func (a *ArangoSyncContainer) GetArgs() []string { + return a.args +} + +func (a *ArangoSyncContainer) GetName() string { + return k8sutil.ServerContainerName } func (a *ArangoSyncContainer) GetPorts() []core.ContainerPort { @@ -293,6 +303,7 @@ func (m *MemberSyncPod) GetContainerCreator() interfaces.ContainerCreator { group: m.group, resources: m.resources, imageInfo: m.imageInfo, + args: m.args, } } diff --git a/pkg/util/k8sutil/interfaces/pod_creator.go b/pkg/util/k8sutil/interfaces/pod_creator.go index 34af9defe..863d87903 100644 --- a/pkg/util/k8sutil/interfaces/pod_creator.go +++ b/pkg/util/k8sutil/interfaces/pod_creator.go @@ -63,6 +63,8 @@ type PodCreator interface { } type ContainerCreator interface { + GetArgs() []string + GetName() string GetExecutor() string GetProbes() (*core.Probe, *core.Probe, error) GetResourceRequirements() core.ResourceRequirements diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index ea0bb3b56..0065a0a31 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -413,7 +413,7 @@ func ExtractPodResourceRequirement(resources core.ResourceRequirements) core.Res } // NewContainer creates a container for specified creator -func NewContainer(args []string, containerCreator interfaces.ContainerCreator) (core.Container, error) { +func NewContainer(containerCreator interfaces.ContainerCreator) (core.Container, error) { liveness, readiness, err := containerCreator.GetProbes() if err != nil { @@ -426,9 +426,9 @@ func NewContainer(args []string, containerCreator interfaces.ContainerCreator) ( } return core.Container{ - Name: ServerContainerName, + Name: containerCreator.GetName(), Image: containerCreator.GetImage(), - Command: append([]string{containerCreator.GetExecutor()}, args...), + Command: append([]string{containerCreator.GetExecutor()}, containerCreator.GetArgs()...), Ports: containerCreator.GetPorts(), Env: containerCreator.GetEnvs(), Resources: containerCreator.GetResourceRequirements(), From bc60acd61f08192c179c4eb755c69d517817c9d8 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Sun, 7 Nov 2021 18:24:30 +0100 Subject: [PATCH 2/9] split volumes and volume mounts functions --- pkg/deployment/images.go | 22 ++-- pkg/deployment/resources/pod_creator.go | 4 +- .../resources/pod_creator_arangod.go | 118 ++++++++++-------- pkg/deployment/resources/pod_creator_sync.go | 68 ++++++---- pkg/util/k8sutil/interfaces/pod_creator.go | 3 +- pkg/util/k8sutil/pods.go | 1 + 6 files changed, 128 insertions(+), 88 deletions(-) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index a4a1e28aa..03631400e 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -284,14 +284,8 @@ func (i *ImageUpdatePod) GetAffinityRole() string { return "" } -func (i *ImageUpdatePod) GetVolumes() ([]core.Volume, []core.VolumeMount) { - var volumes []core.Volume - var volumeMounts []core.VolumeMount - - volumes = append(volumes, k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) - volumeMounts = append(volumeMounts, k8sutil.ArangodVolumeMount()) - - return volumes, volumeMounts +func (i *ImageUpdatePod) GetVolumes() []core.Volume { + return []core.Volume{k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)} } func (i *ImageUpdatePod) GetSidecars(*core.Pod) error { @@ -409,7 +403,7 @@ func (a *ContainerIdentity) GetName() string { func (a *ContainerIdentity) GetPorts() []core.ContainerPort { return []core.ContainerPort{ { - Name: "server", + Name: k8sutil.ServerContainerName, ContainerPort: int32(k8sutil.ArangoPort), Protocol: core.ProtocolTCP, }, @@ -428,6 +422,11 @@ func (a *ContainerIdentity) GetSecurityContext() *core.SecurityContext { return a.ID.Get().SecurityContext.NewSecurityContext() } +// GetVolumeMounts returns nil for the basic container identity. +func (a *ContainerIdentity) GetVolumeMounts() []core.VolumeMount { + return nil +} + func (a *ArangoDIdentity) GetEnvs() []core.EnvVar { env := make([]core.EnvVar, 0) @@ -443,6 +442,11 @@ func (a *ArangoDIdentity) GetEnvs() []core.EnvVar { return nil } +// GetVolumeMounts returns volume mount for the ArangoD data. +func (a *ArangoDIdentity) GetVolumeMounts() []core.VolumeMount { + return []core.VolumeMount{k8sutil.ArangodVolumeMount()} +} + // GetExecutor returns the fixed path to the ArangoSync binary in the container. func (a *ArangoSyncIdentity) GetExecutor() string { return resources.ArangoSyncExecutor diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index a3352cdbe..2acbb114a 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -357,7 +357,6 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect context: r.context, autoUpgrade: autoUpgrade, deploymentStatus: status, - id: memberID, arangoMember: *member, } @@ -368,6 +367,7 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect if err != nil { return nil, errors.WithStack(err) } + memberPod.volumes = CreateArangoDVolumes(*newMember, input, spec, groupSpec) if err := memberPod.Validate(cachedStatus); err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Validation of pods resources failed")) @@ -669,7 +669,7 @@ func RenderArangoPod(cachedStatus inspectorInterface.Inspector, deployment k8sut return nil, errors.WithStack(err) } - p.Spec.Volumes, c.VolumeMounts = podCreator.GetVolumes() + p.Spec.Volumes = podCreator.GetVolumes() p.Spec.Containers = append(p.Spec.Containers, c) if err := podCreator.GetSidecars(&p); err != nil { return nil, err diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index cd831cb15..5c927c10c 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -66,8 +66,8 @@ type MemberArangoDPod struct { resources *Resources imageInfo api.ImageInfo autoUpgrade bool - id string args []string + volumes pod.Volumes } type ArangoDContainer struct { @@ -78,12 +78,13 @@ type ArangoDContainer struct { group api.ServerGroup imageInfo api.ImageInfo args []string + volumes pod.Volumes } func (a *ArangoDContainer) GetPorts() []core.ContainerPort { ports := []core.ContainerPort{ { - Name: "server", + Name: k8sutil.ServerContainerName, ContainerPort: int32(k8sutil.ArangoPort), Protocol: core.ProtocolTCP, }, @@ -215,6 +216,10 @@ func (a *ArangoDContainer) GetImagePullPolicy() core.PullPolicy { return a.spec.GetImagePullPolicy() } +func (a *ArangoDContainer) GetVolumeMounts() []core.VolumeMount { + return a.volumes.VolumeMounts() +} + func (m *MemberArangoDPod) AsInput() pod.Input { return pod.Input{ ApiObject: m.context.GetAPIObject(), @@ -336,55 +341,8 @@ func (m *MemberArangoDPod) GetSidecars(pod *core.Pod) error { return nil } -func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { - volumes := pod.NewVolumes() - - volumes.AddVolumeMount(k8sutil.ArangodVolumeMount()) - - volumes.AddVolumeMount(k8sutil.LifecycleVolumeMount()) - - if m.status.PersistentVolumeClaimName != "" { - vol := k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName, - m.status.PersistentVolumeClaimName) - - volumes.AddVolume(vol) - } else { - volumes.AddVolume(k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) - } - - // TLS - volumes.Append(pod.TLS(), m.AsInput()) - - // Encryption - volumes.Append(pod.Encryption(), m.AsInput()) - - // Security - volumes.Append(pod.Security(), m.AsInput()) - - if m.spec.Metrics.IsEnabled() { - token := m.spec.Metrics.GetJWTTokenSecretName() - if m.spec.Authentication.IsAuthenticated() && token != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token) - volumes.AddVolume(vol) - } - } - - volumes.Append(pod.JWT(), m.AsInput()) - - volumes.AddVolume(k8sutil.LifecycleVolume()) - - // SNI - volumes.Append(pod.SNI(), m.AsInput()) - - if len(m.groupSpec.Volumes) > 0 { - volumes.AddVolume(m.groupSpec.Volumes.Volumes()...) - } - - if len(m.groupSpec.VolumeMounts) > 0 { - volumes.AddVolumeMount(m.groupSpec.VolumeMounts.VolumeMounts()...) - } - - return volumes.Volumes(), volumes.VolumeMounts() +func (m *MemberArangoDPod) GetVolumes() []core.Volume { + return m.volumes.Volumes() } func (m *MemberArangoDPod) IsDeploymentMode() bool { @@ -434,8 +392,6 @@ func (m *MemberArangoDPod) GetInitContainers(cachedStatus interfaces.Inspector) return nil, err } - _, c.VolumeMounts = m.GetVolumes() - c.Name = api.ServerGroupReservedInitContainerNameUpgrade c.Lifecycle = nil c.LivenessProbe = nil @@ -458,8 +414,6 @@ func (m *MemberArangoDPod) GetInitContainers(cachedStatus interfaces.Inspector) return nil, err } - _, c.VolumeMounts = m.GetVolumes() - c.Name = api.ServerGroupReservedInitContainerNameVersionCheck c.Lifecycle = nil c.LivenessProbe = nil @@ -507,6 +461,7 @@ func (m *MemberArangoDPod) GetContainerCreator() interfaces.ContainerCreator { imageInfo: m.imageInfo, groupSpec: m.groupSpec, args: m.args, + volumes: m.volumes, } } @@ -562,3 +517,56 @@ func (m *MemberArangoDPod) Labels() map[string]string { return l } + +// CreateArangoDVolumes returns wrapper with volumes for a pod and volume mounts for a container. +func CreateArangoDVolumes(status api.MemberStatus, input pod.Input, spec api.DeploymentSpec, + groupSpec api.ServerGroupSpec) pod.Volumes { + volumes := pod.NewVolumes() + + volumes.AddVolumeMount(k8sutil.ArangodVolumeMount()) + + volumes.AddVolumeMount(k8sutil.LifecycleVolumeMount()) + + if status.PersistentVolumeClaimName != "" { + vol := k8sutil.CreateVolumeWithPersitantVolumeClaim(k8sutil.ArangodVolumeName, + status.PersistentVolumeClaimName) + + volumes.AddVolume(vol) + } else { + volumes.AddVolume(k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) + } + + // TLS + volumes.Append(pod.TLS(), input) + + // Encryption + volumes.Append(pod.Encryption(), input) + + // Security + volumes.Append(pod.Security(), input) + + if spec.Metrics.IsEnabled() { + token := spec.Metrics.GetJWTTokenSecretName() + if spec.Authentication.IsAuthenticated() && token != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ExporterJWTVolumeName, token) + volumes.AddVolume(vol) + } + } + + volumes.Append(pod.JWT(), input) + + volumes.AddVolume(k8sutil.LifecycleVolume()) + + // SNI + volumes.Append(pod.SNI(), input) + + if len(groupSpec.Volumes) > 0 { + volumes.AddVolume(groupSpec.Volumes.Volumes()...) + } + + if len(groupSpec.VolumeMounts) > 0 { + volumes.AddVolumeMount(groupSpec.VolumeMounts.VolumeMounts()...) + } + + return volumes +} diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index d9477c0dc..6c8072b92 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -43,12 +43,16 @@ const ( ) type ArangoSyncContainer struct { - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - resources *Resources - imageInfo api.ImageInfo - args []string + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + resources *Resources + imageInfo api.ImageInfo + args []string + tlsKeyfileSecretName string + clientAuthCASecretName string + masterJWTSecretName string + clusterJWTSecretName string } var _ interfaces.PodCreator = &MemberSyncPod{} @@ -79,7 +83,7 @@ func (a *ArangoSyncContainer) GetName() string { func (a *ArangoSyncContainer) GetPorts() []core.ContainerPort { return []core.ContainerPort{ { - Name: "server", + Name: k8sutil.ServerContainerName, ContainerPort: int32(k8sutil.ArangoPort), Protocol: core.ProtocolTCP, }, @@ -166,6 +170,30 @@ func (a *ArangoSyncContainer) GetEnvs() []core.EnvVar { return envs.GetEnvList() } +func (a *ArangoSyncContainer) GetVolumeMounts() []core.VolumeMount { + var volumeMounts []core.VolumeMount + + volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount()) + + if a.tlsKeyfileSecretName != "" { + volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount()) + } + + if a.clientAuthCASecretName != "" { + volumeMounts = append(volumeMounts, k8sutil.ClientAuthCACertificateVolumeMount()) + } + + if a.masterJWTSecretName != "" { + volumeMounts = append(volumeMounts, k8sutil.MasterJWTVolumeMount()) + } + + if a.clusterJWTSecretName != "" { + volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount()) + } + + return volumeMounts +} + func (m *MemberSyncPod) GetName() string { return m.resources.context.GetAPIObject().GetName() } @@ -228,41 +256,35 @@ func (m *MemberSyncPod) GetSidecars(pod *core.Pod) error { return nil } -func (m *MemberSyncPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { +func (m *MemberSyncPod) GetVolumes() []core.Volume { var volumes []core.Volume - var volumeMounts []core.VolumeMount volumes = append(volumes, k8sutil.LifecycleVolume()) - volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount()) if m.tlsKeyfileSecretName != "" { vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, m.tlsKeyfileSecretName) volumes = append(volumes, vol) - volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount()) } // Client Authentication certificate secret mount (if any) if m.clientAuthCASecretName != "" { vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClientAuthCAVolumeName, m.clientAuthCASecretName) volumes = append(volumes, vol) - volumeMounts = append(volumeMounts, k8sutil.ClientAuthCACertificateVolumeMount()) } // Master JWT secret mount (if any) if m.masterJWTSecretName != "" { vol := k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, m.masterJWTSecretName) volumes = append(volumes, vol) - volumeMounts = append(volumeMounts, k8sutil.MasterJWTVolumeMount()) } // Cluster JWT secret mount (if any) if m.clusterJWTSecretName != "" { vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, m.clusterJWTSecretName) volumes = append(volumes, vol) - volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount()) } - return volumes, volumeMounts + return volumes } func (m *MemberSyncPod) IsDeploymentMode() bool { @@ -298,12 +320,16 @@ func (m *MemberSyncPod) GetTolerations() []core.Toleration { func (m *MemberSyncPod) GetContainerCreator() interfaces.ContainerCreator { return &ArangoSyncContainer{ - groupSpec: m.groupSpec, - spec: m.spec, - group: m.group, - resources: m.resources, - imageInfo: m.imageInfo, - args: m.args, + groupSpec: m.groupSpec, + spec: m.spec, + group: m.group, + resources: m.resources, + imageInfo: m.imageInfo, + args: m.args, + tlsKeyfileSecretName: m.tlsKeyfileSecretName, + clientAuthCASecretName: m.clientAuthCASecretName, + masterJWTSecretName: m.masterJWTSecretName, + clusterJWTSecretName: m.clusterJWTSecretName, } } diff --git a/pkg/util/k8sutil/interfaces/pod_creator.go b/pkg/util/k8sutil/interfaces/pod_creator.go index 863d87903..cfe4970e0 100644 --- a/pkg/util/k8sutil/interfaces/pod_creator.go +++ b/pkg/util/k8sutil/interfaces/pod_creator.go @@ -41,7 +41,7 @@ type PodCreator interface { Init(*core.Pod) GetName() string GetRole() string - GetVolumes() ([]core.Volume, []core.VolumeMount) + GetVolumes() []core.Volume GetSidecars(*core.Pod) error GetInitContainers(cachedStatus Inspector) ([]core.Container, error) GetFinalizers() []string @@ -74,4 +74,5 @@ type ContainerCreator interface { GetEnvs() []core.EnvVar GetSecurityContext() *core.SecurityContext GetPorts() []core.ContainerPort + GetVolumeMounts() []core.VolumeMount } diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 0065a0a31..8dba03021 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -437,6 +437,7 @@ func NewContainer(containerCreator interfaces.ContainerCreator) (core.Container, Lifecycle: lifecycle, ImagePullPolicy: containerCreator.GetImagePullPolicy(), SecurityContext: containerCreator.GetSecurityContext(), + VolumeMounts: containerCreator.GetVolumeMounts(), }, nil } From 278d1834fadf0fe6c9ab3445cac1975e9872523e Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Sun, 7 Nov 2021 18:51:46 +0100 Subject: [PATCH 3/9] use one place for creation of the arangosync volumes --- pkg/deployment/resources/pod_creator.go | 21 ++- pkg/deployment/resources/pod_creator_sync.go | 139 ++++++++----------- 2 files changed, 69 insertions(+), 91 deletions(-) diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 2acbb114a..ab0120be7 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -428,18 +428,17 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect } // Prepare arguments. + volumes := CreateArangoSyncVolumes(tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, + clusterJWTSecretName) memberSyncPod := MemberSyncPod{ - tlsKeyfileSecretName: tlsKeyfileSecretName, - clientAuthCASecretName: clientAuthCASecretName, - masterJWTSecretName: masterJWTSecretName, - clusterJWTSecretName: clusterJWTSecretName, - groupSpec: groupSpec, - spec: spec, - group: group, - resources: r, - imageInfo: imageInfo, - arangoMember: *member, - args: createArangoSyncArgs(apiObject, spec, group, groupSpec, *newMember), + groupSpec: groupSpec, + spec: spec, + group: group, + resources: r, + imageInfo: imageInfo, + arangoMember: *member, + args: createArangoSyncArgs(apiObject, spec, group, groupSpec, *newMember), + volumes: volumes, } podCreator = &memberSyncPod diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 6c8072b92..714045d7b 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -43,33 +43,27 @@ const ( ) type ArangoSyncContainer struct { - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - resources *Resources - imageInfo api.ImageInfo - args []string - tlsKeyfileSecretName string - clientAuthCASecretName string - masterJWTSecretName string - clusterJWTSecretName string + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + resources *Resources + imageInfo api.ImageInfo + args []string + volumes pod.Volumes } var _ interfaces.PodCreator = &MemberSyncPod{} var _ interfaces.ContainerCreator = &ArangoSyncContainer{} type MemberSyncPod struct { - tlsKeyfileSecretName string - clientAuthCASecretName string - masterJWTSecretName string - clusterJWTSecretName string - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - arangoMember api.ArangoMember - resources *Resources - imageInfo api.ImageInfo - args []string + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + arangoMember api.ArangoMember + resources *Resources + imageInfo api.ImageInfo + args []string + volumes pod.Volumes } func (a *ArangoSyncContainer) GetArgs() []string { @@ -171,27 +165,7 @@ func (a *ArangoSyncContainer) GetEnvs() []core.EnvVar { } func (a *ArangoSyncContainer) GetVolumeMounts() []core.VolumeMount { - var volumeMounts []core.VolumeMount - - volumeMounts = append(volumeMounts, k8sutil.LifecycleVolumeMount()) - - if a.tlsKeyfileSecretName != "" { - volumeMounts = append(volumeMounts, k8sutil.TlsKeyfileVolumeMount()) - } - - if a.clientAuthCASecretName != "" { - volumeMounts = append(volumeMounts, k8sutil.ClientAuthCACertificateVolumeMount()) - } - - if a.masterJWTSecretName != "" { - volumeMounts = append(volumeMounts, k8sutil.MasterJWTVolumeMount()) - } - - if a.clusterJWTSecretName != "" { - volumeMounts = append(volumeMounts, k8sutil.ClusterJWTVolumeMount()) - } - - return volumeMounts + return a.volumes.VolumeMounts() } func (m *MemberSyncPod) GetName() string { @@ -257,34 +231,7 @@ func (m *MemberSyncPod) GetSidecars(pod *core.Pod) error { } func (m *MemberSyncPod) GetVolumes() []core.Volume { - var volumes []core.Volume - - volumes = append(volumes, k8sutil.LifecycleVolume()) - - if m.tlsKeyfileSecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, m.tlsKeyfileSecretName) - volumes = append(volumes, vol) - } - - // Client Authentication certificate secret mount (if any) - if m.clientAuthCASecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClientAuthCAVolumeName, m.clientAuthCASecretName) - volumes = append(volumes, vol) - } - - // Master JWT secret mount (if any) - if m.masterJWTSecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, m.masterJWTSecretName) - volumes = append(volumes, vol) - } - - // Cluster JWT secret mount (if any) - if m.clusterJWTSecretName != "" { - vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, m.clusterJWTSecretName) - volumes = append(volumes, vol) - } - - return volumes + return m.volumes.Volumes() } func (m *MemberSyncPod) IsDeploymentMode() bool { @@ -320,16 +267,13 @@ func (m *MemberSyncPod) GetTolerations() []core.Toleration { func (m *MemberSyncPod) GetContainerCreator() interfaces.ContainerCreator { return &ArangoSyncContainer{ - groupSpec: m.groupSpec, - spec: m.spec, - group: m.group, - resources: m.resources, - imageInfo: m.imageInfo, - args: m.args, - tlsKeyfileSecretName: m.tlsKeyfileSecretName, - clientAuthCASecretName: m.clientAuthCASecretName, - masterJWTSecretName: m.masterJWTSecretName, - clusterJWTSecretName: m.clusterJWTSecretName, + groupSpec: m.groupSpec, + spec: m.spec, + group: m.group, + resources: m.resources, + imageInfo: m.imageInfo, + args: m.args, + volumes: m.volumes, } } @@ -358,3 +302,38 @@ func (m *MemberSyncPod) Annotations() map[string]string { func (m *MemberSyncPod) Labels() map[string]string { return collection.ReservedLabels().Filter(collection.MergeAnnotations(m.spec.Labels, m.groupSpec.Labels)) } + +// CreateArangoSyncVolumes returns wrapper with volumes for a pod and volume mounts for a container. +func CreateArangoSyncVolumes(tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, + clusterJWTSecretName string) pod.Volumes { + volumes := pod.NewVolumes() + + volumes.AddVolume(k8sutil.LifecycleVolume()) + volumes.AddVolumeMount(k8sutil.LifecycleVolumeMount()) + + if tlsKeyfileSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.TlsKeyfileVolumeName, tlsKeyfileSecretName) + volumes.AddVolume(vol) + volumes.AddVolumeMount(k8sutil.TlsKeyfileVolumeMount()) + } + + if clientAuthCASecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClientAuthCAVolumeName, clientAuthCASecretName) + volumes.AddVolume(vol) + volumes.AddVolumeMount(k8sutil.ClientAuthCACertificateVolumeMount()) + } + + if masterJWTSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.MasterJWTSecretVolumeName, masterJWTSecretName) + volumes.AddVolume(vol) + volumes.AddVolumeMount(k8sutil.MasterJWTVolumeMount()) + } + + if clusterJWTSecretName != "" { + vol := k8sutil.CreateVolumeWithSecret(k8sutil.ClusterJWTSecretVolumeName, clusterJWTSecretName) + volumes.AddVolume(vol) + volumes.AddVolumeMount(k8sutil.ClusterJWTVolumeMount()) + } + + return volumes +} From ac067e708c4a589c3c9497b3ce4f1bce56896079 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Sun, 7 Nov 2021 19:06:38 +0100 Subject: [PATCH 4/9] remove unnecessary functions from Volumes interface --- pkg/deployment/pod/volumes.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/pkg/deployment/pod/volumes.go b/pkg/deployment/pod/volumes.go index f4d2564c1..6eba76e2c 100644 --- a/pkg/deployment/pod/volumes.go +++ b/pkg/deployment/pod/volumes.go @@ -32,13 +32,8 @@ func NewVolumes() Volumes { type Volumes interface { Append(b Builder, i Input) - AddVolume(volumes ...core.Volume) - AddVolumes(volumes []core.Volume) - AddVolumeMount(mounts ...core.VolumeMount) - AddVolumeMounts(mounts []core.VolumeMount) - Volumes() []core.Volume VolumeMounts() []core.VolumeMount } @@ -52,15 +47,11 @@ type volumes struct { func (v *volumes) Append(b Builder, i Input) { vols, mounts := b.Volumes(i) - v.AddVolumes(vols) - v.AddVolumeMounts(mounts) + v.AddVolume(vols...) + v.AddVolumeMount(mounts...) } func (v *volumes) AddVolume(volumes ...core.Volume) { - v.AddVolumes(volumes) -} - -func (v *volumes) AddVolumes(volumes []core.Volume) { if len(volumes) == 0 { return } @@ -69,10 +60,6 @@ func (v *volumes) AddVolumes(volumes []core.Volume) { } func (v *volumes) AddVolumeMount(mounts ...core.VolumeMount) { - v.AddVolumeMounts(mounts) -} - -func (v *volumes) AddVolumeMounts(mounts []core.VolumeMount) { if len(mounts) == 0 { return } From ddcf5b28247b98459ab40140ff458ae5e8b843e1 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Mon, 8 Nov 2021 12:01:28 +0100 Subject: [PATCH 5/9] create args container dynamically --- pkg/deployment/images.go | 20 ++- pkg/deployment/resources/pod_creator.go | 13 +- .../resources/pod_creator_arangod.go | 125 ++++++++++++------ pkg/deployment/resources/pod_creator_sync.go | 39 +++--- pkg/util/k8sutil/interfaces/pod_creator.go | 2 +- pkg/util/k8sutil/pods.go | 7 +- 6 files changed, 130 insertions(+), 76 deletions(-) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 03631400e..9d3a527ce 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -54,7 +54,7 @@ type ImageUpdatePod struct { // ContainerIdentity helps to resolve the container identity, e.g.: image ID, version of the entrypoint. type ContainerIdentity struct { - args []string + ipAddress string ID *api.ServerIDGroupSpec image string imagePullPolicy core.PullPolicy @@ -210,21 +210,14 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, cac Msg("Found image ID and ArangoDB version") return false, nil } - // Pod cannot be fetched, ensure it is created - args := []string{ - "--server.authentication=false", - fmt.Sprintf("--server.endpoint=tcp://%s:%d", ib.Spec.GetListenAddr(), k8sutil.ArangoPort), - "--database.directory=" + k8sutil.ArangodVolumeMountDir, - "--log.output=+", - } imagePod := ImageUpdatePod{ spec: ib.Spec, apiObject: ib.APIObject, containerCreator: &ArangoDIdentity{ ContainerCreator: &ContainerIdentity{ - args: args, ID: ib.Spec.ID, + ipAddress: ib.Spec.GetListenAddr(), image: image, imagePullPolicy: ib.Spec.GetImagePullPolicy(), }, @@ -372,8 +365,13 @@ func (i *ImageUpdatePod) ApplyPodSpec(_ *core.PodSpec) error { return nil } -func (a *ContainerIdentity) GetArgs() []string { - return a.args +func (a *ContainerIdentity) GetArgs() ([]string, error) { + return []string{ + "--server.authentication=false", + fmt.Sprintf("--server.endpoint=tcp://%s:%d", a.ipAddress, k8sutil.ArangoPort), + "--database.directory=" + k8sutil.ArangodVolumeMountDir, + "--log.output=+", + }, nil } func (a *ContainerIdentity) GetEnvs() []core.EnvVar { diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index ab0120be7..8a69861ed 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -180,7 +180,8 @@ func createArangodArgs(cachedStatus interfaces.Inspector, input pod.Input, addit } // createArangoSyncArgs creates command line arguments for an arangosync server in the given group. -func createArangoSyncArgs(apiObject meta.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, member api.MemberStatus) []string { +func createArangoSyncArgs(apiObject meta.Object, spec api.DeploymentSpec, group api.ServerGroup, + groupSpec api.ServerGroupSpec, member api.MemberStatus) []string { options := k8sutil.CreateOptionPairs(64) var runCmd string var port int @@ -358,15 +359,10 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect autoUpgrade: autoUpgrade, deploymentStatus: status, arangoMember: *member, + cachedStatus: cachedStatus, } input := memberPod.AsInput() - - var err error - memberPod.args, err = createArangodArgs(cachedStatus, input) - if err != nil { - return nil, errors.WithStack(err) - } memberPod.volumes = CreateArangoDVolumes(*newMember, input, spec, groupSpec) if err := memberPod.Validate(cachedStatus); err != nil { @@ -437,7 +433,8 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect resources: r, imageInfo: imageInfo, arangoMember: *member, - args: createArangoSyncArgs(apiObject, spec, group, groupSpec, *newMember), + apiObject: apiObject, + memberStatus: *newMember, volumes: volumes, } diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index 5c927c10c..ff03a480c 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -66,19 +66,35 @@ type MemberArangoDPod struct { resources *Resources imageInfo api.ImageInfo autoUpgrade bool - args []string + cachedStatus interfaces.Inspector volumes pod.Volumes } type ArangoDContainer struct { - member *MemberArangoDPod - resources *Resources - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - imageInfo api.ImageInfo - args []string - volumes pod.Volumes + member *MemberArangoDPod + resources *Resources + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + imageInfo api.ImageInfo + cachedStatus interfaces.Inspector + input pod.Input + volumes pod.Volumes +} + +// ArangoUpgradeContainer can construct ArangoD upgrade container. +type ArangoUpgradeContainer struct { + interfaces.ContainerCreator + cachedStatus interfaces.Inspector + input pod.Input +} + +// ArangoVersionCheckContainer can construct ArangoD version check container. +type ArangoVersionCheckContainer struct { + interfaces.ContainerCreator + cachedStatus interfaces.Inspector + input pod.Input + versionArgs k8sutil.OptionPairs } func (a *ArangoDContainer) GetPorts() []core.ContainerPort { @@ -104,8 +120,8 @@ func (a *ArangoDContainer) GetPorts() []core.ContainerPort { return ports } -func (a *ArangoDContainer) GetArgs() []string { - return a.args +func (a *ArangoDContainer) GetArgs() ([]string, error) { + return createArangodArgs(a.cachedStatus, a.input) } func (a *ArangoDContainer) GetName() string { @@ -382,21 +398,16 @@ func (m *MemberArangoDPod) GetInitContainers(cachedStatus interfaces.Inspector) { // Upgrade container - run in background if m.autoUpgrade || m.status.Upgrade { - m.args, err = createArangodArgsWithUpgrade(cachedStatus, m.AsInput()) - if err != nil { - return nil, err + upgradeContainer := &ArangoUpgradeContainer{ + m.GetContainerCreator(), + cachedStatus, + m.AsInput(), } - - c, err := k8sutil.NewContainer(m.GetContainerCreator()) + c, err := k8sutil.NewContainer(upgradeContainer) if err != nil { return nil, err } - c.Name = api.ServerGroupReservedInitContainerNameUpgrade - c.Lifecycle = nil - c.LivenessProbe = nil - c.ReadinessProbe = nil - initContainers = append(initContainers, c) } @@ -404,21 +415,18 @@ func (m *MemberArangoDPod) GetInitContainers(cachedStatus interfaces.Inspector) { versionArgs := pod.UpgradeVersionCheck().Args(m.AsInput()) if len(versionArgs) > 0 { - m.args, err = createArangodArgs(cachedStatus, m.AsInput(), versionArgs...) - if err != nil { - return nil, err + upgradeContainer := &ArangoVersionCheckContainer{ + m.GetContainerCreator(), + cachedStatus, + m.AsInput(), + versionArgs, } - c, err := k8sutil.NewContainer(m.GetContainerCreator()) + c, err := k8sutil.NewContainer(upgradeContainer) if err != nil { return nil, err } - c.Name = api.ServerGroupReservedInitContainerNameVersionCheck - c.Lifecycle = nil - c.LivenessProbe = nil - c.ReadinessProbe = nil - initContainers = append(initContainers, c) } } @@ -454,14 +462,15 @@ func (m *MemberArangoDPod) GetTolerations() []core.Toleration { func (m *MemberArangoDPod) GetContainerCreator() interfaces.ContainerCreator { return &ArangoDContainer{ - member: m, - spec: m.spec, - group: m.group, - resources: m.resources, - imageInfo: m.imageInfo, - groupSpec: m.groupSpec, - args: m.args, - volumes: m.volumes, + member: m, + spec: m.spec, + group: m.group, + resources: m.resources, + imageInfo: m.imageInfo, + groupSpec: m.groupSpec, + volumes: m.volumes, + cachedStatus: m.cachedStatus, + input: m.AsInput(), } } @@ -570,3 +579,43 @@ func CreateArangoDVolumes(status api.MemberStatus, input pod.Input, spec api.Dep return volumes } + +// GetArgs returns list of arguments for the ArangoD upgrade container. +func (a *ArangoUpgradeContainer) GetArgs() ([]string, error) { + return createArangodArgsWithUpgrade(a.cachedStatus, a.input) +} + +// GetLifecycle returns no lifecycle for the ArangoD upgrade container. +func (a *ArangoUpgradeContainer) GetLifecycle() (*core.Lifecycle, error) { + return nil, nil +} + +// GetName returns the name of the ArangoD upgrade container. +func (a *ArangoUpgradeContainer) GetName() string { + return api.ServerGroupReservedInitContainerNameUpgrade +} + +// GetProbes returns no probes for the ArangoD upgrade container. +func (a *ArangoUpgradeContainer) GetProbes() (*core.Probe, *core.Probe, error) { + return nil, nil, nil +} + +// GetArgs returns list of arguments for the ArangoD version check container. +func (a *ArangoVersionCheckContainer) GetArgs() ([]string, error) { + return createArangodArgs(a.cachedStatus, a.input, a.versionArgs...) +} + +// GetLifecycle returns no lifecycle for the ArangoD version check container. +func (a *ArangoVersionCheckContainer) GetLifecycle() (*core.Lifecycle, error) { + return nil, nil +} + +// GetName returns the name of the ArangoD version check container. +func (a *ArangoVersionCheckContainer) GetName() string { + return api.ServerGroupReservedInitContainerNameVersionCheck +} + +// GetProbes returns no probes for the ArangoD version check container. +func (a *ArangoVersionCheckContainer) GetProbes() (*core.Probe, *core.Probe, error) { + return nil, nil, nil +} diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 714045d7b..b01e94c47 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -23,6 +23,7 @@ package resources import ( + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "math" "github.com/arangodb/kube-arangodb/pkg/util/collection" @@ -43,13 +44,15 @@ const ( ) type ArangoSyncContainer struct { - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - resources *Resources - imageInfo api.ImageInfo - args []string - volumes pod.Volumes + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + resources *Resources + imageInfo api.ImageInfo + apiObject meta.Object + memberStatus api.MemberStatus + args []string + volumes pod.Volumes } var _ interfaces.PodCreator = &MemberSyncPod{} @@ -62,12 +65,13 @@ type MemberSyncPod struct { arangoMember api.ArangoMember resources *Resources imageInfo api.ImageInfo - args []string + apiObject meta.Object + memberStatus api.MemberStatus volumes pod.Volumes } -func (a *ArangoSyncContainer) GetArgs() []string { - return a.args +func (a *ArangoSyncContainer) GetArgs() ([]string, error) { + return createArangoSyncArgs(a.apiObject, a.spec, a.group, a.groupSpec, a.memberStatus), nil } func (a *ArangoSyncContainer) GetName() string { @@ -267,13 +271,14 @@ func (m *MemberSyncPod) GetTolerations() []core.Toleration { func (m *MemberSyncPod) GetContainerCreator() interfaces.ContainerCreator { return &ArangoSyncContainer{ - groupSpec: m.groupSpec, - spec: m.spec, - group: m.group, - resources: m.resources, - imageInfo: m.imageInfo, - args: m.args, - volumes: m.volumes, + groupSpec: m.groupSpec, + spec: m.spec, + group: m.group, + resources: m.resources, + imageInfo: m.imageInfo, + apiObject: m.apiObject, + memberStatus: m.memberStatus, + volumes: m.volumes, } } diff --git a/pkg/util/k8sutil/interfaces/pod_creator.go b/pkg/util/k8sutil/interfaces/pod_creator.go index cfe4970e0..1cd908071 100644 --- a/pkg/util/k8sutil/interfaces/pod_creator.go +++ b/pkg/util/k8sutil/interfaces/pod_creator.go @@ -63,7 +63,7 @@ type PodCreator interface { } type ContainerCreator interface { - GetArgs() []string + GetArgs() ([]string, error) GetName() string GetExecutor() string GetProbes() (*core.Probe, *core.Probe, error) diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 8dba03021..9aefe9b50 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -425,10 +425,15 @@ func NewContainer(containerCreator interfaces.ContainerCreator) (core.Container, return core.Container{}, err } + args, err := containerCreator.GetArgs() + if err != nil { + return core.Container{}, err + } + return core.Container{ Name: containerCreator.GetName(), Image: containerCreator.GetImage(), - Command: append([]string{containerCreator.GetExecutor()}, containerCreator.GetArgs()...), + Command: append([]string{containerCreator.GetExecutor()}, args...), Ports: containerCreator.GetPorts(), Env: containerCreator.GetEnvs(), Resources: containerCreator.GetResourceRequirements(), From 224677375e52c15ac5244b973c1028f7dc6643c8 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Mon, 8 Nov 2021 12:16:41 +0100 Subject: [PATCH 6/9] ArangoDIdentity uses dynamic volumes and mounts --- pkg/deployment/images.go | 12 ++++++++++-- pkg/deployment/resources/pod_creator.go | 2 +- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 9d3a527ce..8ba1b8e94 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -278,7 +278,7 @@ func (i *ImageUpdatePod) GetAffinityRole() string { } func (i *ImageUpdatePod) GetVolumes() []core.Volume { - return []core.Volume{k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)} + return getVolumes().Volumes() } func (i *ImageUpdatePod) GetSidecars(*core.Pod) error { @@ -442,10 +442,18 @@ func (a *ArangoDIdentity) GetEnvs() []core.EnvVar { // GetVolumeMounts returns volume mount for the ArangoD data. func (a *ArangoDIdentity) GetVolumeMounts() []core.VolumeMount { - return []core.VolumeMount{k8sutil.ArangodVolumeMount()} + return getVolumes().VolumeMounts() } // GetExecutor returns the fixed path to the ArangoSync binary in the container. func (a *ArangoSyncIdentity) GetExecutor() string { return resources.ArangoSyncExecutor } + +func getVolumes() pod.Volumes { + volumes := pod.NewVolumes() + volumes.AddVolume(k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName)) + volumes.AddVolumeMount(k8sutil.ArangodVolumeMount()) + + return volumes +} diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 8a69861ed..6855d82b0 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -660,12 +660,12 @@ func RenderArangoPod(cachedStatus inspectorInterface.Inspector, deployment k8sut p.Spec.InitContainers = append(p.Spec.InitContainers, initContainers...) } + p.Spec.Volumes = podCreator.GetVolumes() c, err := k8sutil.NewContainer(podCreator.GetContainerCreator()) if err != nil { return nil, errors.WithStack(err) } - p.Spec.Volumes = podCreator.GetVolumes() p.Spec.Containers = append(p.Spec.Containers, c) if err := podCreator.GetSidecars(&p); err != nil { return nil, err From c37e69c6df1192c6ae07f41cae7985371937f6eb Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Mon, 8 Nov 2021 13:00:34 +0100 Subject: [PATCH 7/9] create container arangod volumes dynamically --- pkg/deployment/resources/pod_creator.go | 31 ++++---- .../resources/pod_creator_arangod.go | 13 ++-- pkg/deployment/resources/pod_creator_sync.go | 74 +++++++++++-------- 3 files changed, 65 insertions(+), 53 deletions(-) diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 6855d82b0..d3243a995 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -362,9 +362,6 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect cachedStatus: cachedStatus, } - input := memberPod.AsInput() - memberPod.volumes = CreateArangoDVolumes(*newMember, input, spec, groupSpec) - if err := memberPod.Validate(cachedStatus); err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Validation of pods resources failed")) } @@ -423,22 +420,20 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect } } - // Prepare arguments. - volumes := CreateArangoSyncVolumes(tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, - clusterJWTSecretName) - memberSyncPod := MemberSyncPod{ - groupSpec: groupSpec, - spec: spec, - group: group, - resources: r, - imageInfo: imageInfo, - arangoMember: *member, - apiObject: apiObject, - memberStatus: *newMember, - volumes: volumes, + podCreator = &MemberSyncPod{ + tlsKeyfileSecretName: tlsKeyfileSecretName, + clientAuthCASecretName: clientAuthCASecretName, + masterJWTSecretName: masterJWTSecretName, + clusterJWTSecretName: clusterJWTSecretName, + groupSpec: groupSpec, + spec: spec, + group: group, + resources: r, + imageInfo: imageInfo, + arangoMember: *member, + apiObject: apiObject, + memberStatus: *newMember, } - - podCreator = &memberSyncPod } else { return nil, errors.Newf("unable to render Pod") } diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index ff03a480c..e20f81ba2 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -67,7 +67,6 @@ type MemberArangoDPod struct { imageInfo api.ImageInfo autoUpgrade bool cachedStatus interfaces.Inspector - volumes pod.Volumes } type ArangoDContainer struct { @@ -79,7 +78,7 @@ type ArangoDContainer struct { imageInfo api.ImageInfo cachedStatus interfaces.Inspector input pod.Input - volumes pod.Volumes + status api.MemberStatus } // ArangoUpgradeContainer can construct ArangoD upgrade container. @@ -233,7 +232,9 @@ func (a *ArangoDContainer) GetImagePullPolicy() core.PullPolicy { } func (a *ArangoDContainer) GetVolumeMounts() []core.VolumeMount { - return a.volumes.VolumeMounts() + volumes := CreateArangoDVolumes(a.status, a.input, a.spec, a.groupSpec) + + return volumes.VolumeMounts() } func (m *MemberArangoDPod) AsInput() pod.Input { @@ -358,7 +359,9 @@ func (m *MemberArangoDPod) GetSidecars(pod *core.Pod) error { } func (m *MemberArangoDPod) GetVolumes() []core.Volume { - return m.volumes.Volumes() + volumes := CreateArangoDVolumes(m.status, m.AsInput(), m.spec, m.groupSpec) + + return volumes.Volumes() } func (m *MemberArangoDPod) IsDeploymentMode() bool { @@ -468,9 +471,9 @@ func (m *MemberArangoDPod) GetContainerCreator() interfaces.ContainerCreator { resources: m.resources, imageInfo: m.imageInfo, groupSpec: m.groupSpec, - volumes: m.volumes, cachedStatus: m.cachedStatus, input: m.AsInput(), + status: m.status, } } diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index b01e94c47..9d57515e5 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -44,30 +44,35 @@ const ( ) type ArangoSyncContainer struct { - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - resources *Resources - imageInfo api.ImageInfo - apiObject meta.Object - memberStatus api.MemberStatus - args []string - volumes pod.Volumes + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + resources *Resources + imageInfo api.ImageInfo + apiObject meta.Object + memberStatus api.MemberStatus + tlsKeyfileSecretName string + clientAuthCASecretName string + masterJWTSecretName string + clusterJWTSecretName string } var _ interfaces.PodCreator = &MemberSyncPod{} var _ interfaces.ContainerCreator = &ArangoSyncContainer{} type MemberSyncPod struct { - groupSpec api.ServerGroupSpec - spec api.DeploymentSpec - group api.ServerGroup - arangoMember api.ArangoMember - resources *Resources - imageInfo api.ImageInfo - apiObject meta.Object - memberStatus api.MemberStatus - volumes pod.Volumes + tlsKeyfileSecretName string + clientAuthCASecretName string + masterJWTSecretName string + clusterJWTSecretName string + groupSpec api.ServerGroupSpec + spec api.DeploymentSpec + group api.ServerGroup + arangoMember api.ArangoMember + resources *Resources + imageInfo api.ImageInfo + apiObject meta.Object + memberStatus api.MemberStatus } func (a *ArangoSyncContainer) GetArgs() ([]string, error) { @@ -169,7 +174,10 @@ func (a *ArangoSyncContainer) GetEnvs() []core.EnvVar { } func (a *ArangoSyncContainer) GetVolumeMounts() []core.VolumeMount { - return a.volumes.VolumeMounts() + volumes := createArangoSyncVolumes(a.tlsKeyfileSecretName, a.clientAuthCASecretName, a.masterJWTSecretName, + a.clusterJWTSecretName) + + return volumes.VolumeMounts() } func (m *MemberSyncPod) GetName() string { @@ -234,8 +242,12 @@ func (m *MemberSyncPod) GetSidecars(pod *core.Pod) error { return nil } +// GetVolumes returns volumes for the ArangoSync container. func (m *MemberSyncPod) GetVolumes() []core.Volume { - return m.volumes.Volumes() + volumes := createArangoSyncVolumes(m.tlsKeyfileSecretName, m.clientAuthCASecretName, m.masterJWTSecretName, + m.clusterJWTSecretName) + + return volumes.Volumes() } func (m *MemberSyncPod) IsDeploymentMode() bool { @@ -271,14 +283,17 @@ func (m *MemberSyncPod) GetTolerations() []core.Toleration { func (m *MemberSyncPod) GetContainerCreator() interfaces.ContainerCreator { return &ArangoSyncContainer{ - groupSpec: m.groupSpec, - spec: m.spec, - group: m.group, - resources: m.resources, - imageInfo: m.imageInfo, - apiObject: m.apiObject, - memberStatus: m.memberStatus, - volumes: m.volumes, + groupSpec: m.groupSpec, + spec: m.spec, + group: m.group, + resources: m.resources, + imageInfo: m.imageInfo, + apiObject: m.apiObject, + memberStatus: m.memberStatus, + tlsKeyfileSecretName: m.tlsKeyfileSecretName, + clientAuthCASecretName: m.clientAuthCASecretName, + masterJWTSecretName: m.masterJWTSecretName, + clusterJWTSecretName: m.clusterJWTSecretName, } } @@ -308,8 +323,7 @@ func (m *MemberSyncPod) Labels() map[string]string { return collection.ReservedLabels().Filter(collection.MergeAnnotations(m.spec.Labels, m.groupSpec.Labels)) } -// CreateArangoSyncVolumes returns wrapper with volumes for a pod and volume mounts for a container. -func CreateArangoSyncVolumes(tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, +func createArangoSyncVolumes(tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string) pod.Volumes { volumes := pod.NewVolumes() From 9a14f7faf5290a0f05f2e971fb11e19fda1447e1 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Mon, 8 Nov 2021 13:01:43 +0100 Subject: [PATCH 8/9] fix fmt --- pkg/deployment/resources/pod_creator_sync.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 9d57515e5..8dbaf7b03 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -23,9 +23,10 @@ package resources import ( - meta "k8s.io/apimachinery/pkg/apis/meta/v1" "math" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/arangodb/kube-arangodb/pkg/util/collection" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/interfaces" From 135d22373b721cf75e534fdb15d2582de10a33bd Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Mon, 8 Nov 2021 13:20:37 +0100 Subject: [PATCH 9/9] move ArangoDIdentity args to the proper place --- pkg/deployment/images.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 8ba1b8e94..ee31f5937 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -54,7 +54,6 @@ type ImageUpdatePod struct { // ContainerIdentity helps to resolve the container identity, e.g.: image ID, version of the entrypoint. type ContainerIdentity struct { - ipAddress string ID *api.ServerIDGroupSpec image string imagePullPolicy core.PullPolicy @@ -63,7 +62,8 @@ type ContainerIdentity struct { // ArangoDIdentity helps to resolve the ArangoD identity, e.g.: image ID, version of the entrypoint. type ArangoDIdentity struct { interfaces.ContainerCreator - License api.LicenseSpec + License api.LicenseSpec + ipAddress string } // ArangoSyncIdentity helps to resolve the ArangoSync identity, e.g.: image ID, version of the entrypoint. @@ -217,11 +217,11 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, cac containerCreator: &ArangoDIdentity{ ContainerCreator: &ContainerIdentity{ ID: ib.Spec.ID, - ipAddress: ib.Spec.GetListenAddr(), image: image, imagePullPolicy: ib.Spec.GetImagePullPolicy(), }, - License: ib.Spec.License, + License: ib.Spec.License, + ipAddress: ib.Spec.GetListenAddr(), }, } @@ -366,12 +366,7 @@ func (i *ImageUpdatePod) ApplyPodSpec(_ *core.PodSpec) error { } func (a *ContainerIdentity) GetArgs() ([]string, error) { - return []string{ - "--server.authentication=false", - fmt.Sprintf("--server.endpoint=tcp://%s:%d", a.ipAddress, k8sutil.ArangoPort), - "--database.directory=" + k8sutil.ArangodVolumeMountDir, - "--log.output=+", - }, nil + return nil, nil } func (a *ContainerIdentity) GetEnvs() []core.EnvVar { @@ -425,6 +420,16 @@ func (a *ContainerIdentity) GetVolumeMounts() []core.VolumeMount { return nil } +// GetArgs returns the list of arguments for the ArangoD container identification. +func (a *ArangoDIdentity) GetArgs() ([]string, error) { + return []string{ + "--server.authentication=false", + fmt.Sprintf("--server.endpoint=tcp://%s:%d", a.ipAddress, k8sutil.ArangoPort), + "--database.directory=" + k8sutil.ArangodVolumeMountDir, + "--log.output=+", + }, nil +} + func (a *ArangoDIdentity) GetEnvs() []core.EnvVar { env := make([]core.EnvVar, 0)