diff --git a/go.mod b/go.mod index a5ab346bd..20af8826d 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 // indirect github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 // indirect github.com/voxelbrain/goptions v0.0.0-20180630082107-58cddc247ea2 // indirect + golang.org/x/net v0.0.0-20200226121028-0de0cce0169b golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a // indirect golang.org/x/sys v0.0.0-20200116001909-b77594299b42 golang.org/x/tools v0.0.0-20200331202046-9d5940d49312 // indirect diff --git a/pkg/apis/deployment/v1/plan.go b/pkg/apis/deployment/v1/plan.go index f49614082..4087d1b1a 100644 --- a/pkg/apis/deployment/v1/plan.go +++ b/pkg/apis/deployment/v1/plan.go @@ -61,6 +61,8 @@ const ( ActionTypeRenewTLSCertificate ActionType = "RenewTLSCertificate" // ActionTypeRenewTLSCACertificate causes the TLS CA certificate of the entire deployment to be renewed. ActionTypeRenewTLSCACertificate ActionType = "RenewTLSCACertificate" + // ActionTypeUpdateTLSSNI update SNI inplace. + ActionTypeUpdateTLSSNI ActionType = "UpdateTLSSNI" // ActionTypeSetCurrentImage causes status.CurrentImage to be updated to the image given in the action. ActionTypeSetCurrentImage ActionType = "SetCurrentImage" // ActionTypeDisableClusterScaling turns off scaling DBservers and coordinators diff --git a/pkg/apis/deployment/v1/tls_sni_spec.go b/pkg/apis/deployment/v1/tls_sni_spec.go new file mode 100644 index 000000000..06218413d --- /dev/null +++ b/pkg/apis/deployment/v1/tls_sni_spec.go @@ -0,0 +1,81 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Ewout Prangsma +// + +package v1 + +import ( + shared "github.com/arangodb/kube-arangodb/pkg/apis/shared/v1" + "github.com/pkg/errors" +) + +type TLSSNIRotateMode string + +func (t *TLSSNIRotateMode) Get() TLSSNIRotateMode { + if t == nil { + return TLSSNIRotateModeInPlace + } + + return *t +} + +const ( + TLSSNIRotateModeInPlace TLSSNIRotateMode = "inplace" + TLSSNIRotateModeRecreate TLSSNIRotateMode = "recreate" +) + +// TLSSNISpec holds TLS SNI additional certificates +type TLSSNISpec struct { + Mapping map[string][]string `json:"sniMapping,omitempty"` + Mode *TLSSNIRotateMode `json:"mode,omitempty"` +} + +func (s TLSSNISpec) Validate() error { + mapped := map[string]interface{}{} + + for key, values := range s.Mapping { + if err := shared.IsValidName(key); err != nil { + return err + } + + for _, value := range values { + if _, exists := mapped[value]; exists { + return errors.Errorf("sni for host %s is already defined", value) + } + + // Mark value as existing + mapped[value] = nil + + if err := shared.IsValidDomain(value); err != nil { + return err + } + } + } + + return nil +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *TLSSNISpec) SetDefaultsFrom(source *TLSSNISpec) { + if source == nil { + return + } +} diff --git a/pkg/apis/deployment/v1/tls_spec.go b/pkg/apis/deployment/v1/tls_spec.go index 47aa2643e..96e329d4a 100644 --- a/pkg/apis/deployment/v1/tls_spec.go +++ b/pkg/apis/deployment/v1/tls_spec.go @@ -37,9 +37,10 @@ const ( // TLSSpec holds TLS specific configuration settings type TLSSpec struct { - CASecretName *string `json:"caSecretName,omitempty"` - AltNames []string `json:"altNames,omitempty"` - TTL *Duration `json:"ttl,omitempty"` + CASecretName *string `json:"caSecretName,omitempty"` + AltNames []string `json:"altNames,omitempty"` + TTL *Duration `json:"ttl,omitempty"` + SNI *TLSSNISpec `json:",inline"` } const ( @@ -57,6 +58,14 @@ func (s TLSSpec) GetAltNames() []string { return s.AltNames } +func (s TLSSpec) GetTLSSNISpec() TLSSNISpec { + if s.SNI == nil { + return TLSSNISpec{} + } + + return *s.SNI +} + // GetTTL returns the value of ttl. func (s TLSSpec) GetTTL() Duration { return DurationOrDefault(s.TTL) @@ -125,4 +134,6 @@ func (s *TLSSpec) SetDefaultsFrom(source TLSSpec) { if s.TTL == nil { s.TTL = NewDurationOrNil(source.TTL) } + + s.SNI.SetDefaultsFrom(source.SNI) } diff --git a/pkg/apis/deployment/v1/zz_generated.deepcopy.go b/pkg/apis/deployment/v1/zz_generated.deepcopy.go index 920d8da91..bb64788fb 100644 --- a/pkg/apis/deployment/v1/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1/zz_generated.deepcopy.go @@ -1308,6 +1308,42 @@ func (in *SyncSpec) DeepCopy() *SyncSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TLSSNISpec) DeepCopyInto(out *TLSSNISpec) { + *out = *in + if in.Mapping != nil { + in, out := &in.Mapping, &out.Mapping + *out = make(map[string][]string, len(*in)) + for key, val := range *in { + var outVal []string + if val == nil { + (*out)[key] = nil + } else { + in, out := &val, &outVal + *out = make([]string, len(*in)) + copy(*out, *in) + } + (*out)[key] = outVal + } + } + if in.Mode != nil { + in, out := &in.Mode, &out.Mode + *out = new(TLSSNIRotateMode) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TLSSNISpec. +func (in *TLSSNISpec) DeepCopy() *TLSSNISpec { + if in == nil { + return nil + } + out := new(TLSSNISpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { *out = *in @@ -1326,6 +1362,11 @@ func (in *TLSSpec) DeepCopyInto(out *TLSSpec) { *out = new(Duration) **out = **in } + if in.SNI != nil { + in, out := &in.SNI, &out.SNI + *out = new(TLSSNISpec) + (*in).DeepCopyInto(*out) + } return } diff --git a/pkg/apis/shared/v1/resource.go b/pkg/apis/shared/v1/resource.go index 3ba498e48..71e17e62e 100644 --- a/pkg/apis/shared/v1/resource.go +++ b/pkg/apis/shared/v1/resource.go @@ -110,3 +110,11 @@ func IsValidName(name string) error { return nil } + +func IsValidDomain(name string) error { + if res := validation.IsDNS1123Subdomain(name); len(res) > 0 { + return errors.Errorf("validation of domain failed: %s", strings.Join(res, ", ")) + } + + return nil +} diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index bf9ee9a46..98613f797 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -76,7 +76,7 @@ func (d *Deployment) GetAlpineImage() string { return d.config.AlpineImage } -// GetNamespace returns the kubernetes namespace that contains +// GetNamespSecretsInterfaceace returns the kubernetes namespace that contains // this deployment. func (d *Deployment) GetNamespace() string { return d.apiObject.GetNamespace() @@ -438,12 +438,6 @@ func (d *Deployment) DeleteSecret(secretName string) error { return nil } -// GetExpectedPodArguments creates command line arguments for a server in the given group with given ID. -func (d *Deployment) GetExpectedPodArguments(apiObject metav1.Object, deplSpec api.DeploymentSpec, group api.ServerGroup, - agents api.MemberStatusList, id string, version driver.Version) []string { - return d.resources.GetExpectedPodArguments(apiObject, deplSpec, group, agents, id, version) -} - // GetShardSyncStatus returns true if all shards are in sync func (d *Deployment) GetShardSyncStatus() bool { return d.resources.GetShardSyncStatus() @@ -506,3 +500,7 @@ func (d *Deployment) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, return d.updateStatus(status, version, force...) } + +func (d *Deployment) SecretsInterface() k8sutil.SecretInterface { + return d.GetKubeCli().CoreV1().Secrets(d.GetNamespace()) +} diff --git a/pkg/deployment/deployment_core_test.go b/pkg/deployment/deployment_core_test.go index fed6859f5..ffde13448 100644 --- a/pkg/deployment/deployment_core_test.go +++ b/pkg/deployment/deployment_core_test.go @@ -1240,8 +1240,6 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe(true, authorization, k8sutil.ArangoPort) - testCase.ExpectedPod.Spec.Containers[1].VolumeMounts = append( - testCase.ExpectedPod.Spec.Containers[1].VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) }, config: Config{ LifecycleImage: testImageLifecycle, @@ -1287,7 +1285,11 @@ func TestEnsurePod_ArangoDB_Core(t *testing.T) { }, Resources: emptyResources, }, - testCreateExporterContainer(true, emptyResources), + func() core.Container { + c := testCreateExporterContainer(true, emptyResources) + c.VolumeMounts = append(c.VolumeMounts, k8sutil.TlsKeyfileVolumeMount()) + return c + }(), }, RestartPolicy: core.RestartPolicyNever, TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index a176317d0..3a85974bb 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -179,7 +179,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Create scale/update plan - if err, updated := d.reconciler.CreatePlan(); err != nil { + if err, updated := d.reconciler.CreatePlan(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Plan creation failed") } else if updated { return minInspectionInterval, nil diff --git a/pkg/deployment/deployment_pod_tls_sni_test.go b/pkg/deployment/deployment_pod_tls_sni_test.go new file mode 100644 index 000000000..b7a61935a --- /dev/null +++ b/pkg/deployment/deployment_pod_tls_sni_test.go @@ -0,0 +1,460 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package deployment + +import ( + "fmt" + "testing" + + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/stretchr/testify/require" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" +) + +func createTLSSNISecret(t *testing.T, client kubernetes.Interface, name, namespace, key, value string) { + secret := core.Secret{ + ObjectMeta: meta.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Type: core.SecretTypeOpaque, + Data: map[string][]byte{}, + } + + if key != "" { + secret.Data[key] = []byte(value) + } + + _, err := client.CoreV1().Secrets(namespace).Create(&secret) + require.NoError(t, err) +} + +func TestEnsurePod_ArangoDB_TLS_SNI(t *testing.T) { + testCases := []testCaseStruct{ + { + Name: "Pod SNI Mounts", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: func() api.TLSSpec { + s := tlsSpec.DeepCopy() + + s.SNI = &api.TLSSNISpec{ + Mapping: map[string][]string{ + "sni1": { + "a", + "b", + }, + "sni2": { + "c", + "d", + }, + }, + } + + return *s + }(), + }, + }, + Resources: func(t *testing.T, deployment *Deployment) { + createTLSSNISecret(t, deployment.GetKubeCli(), "sni1", deployment.Namespace(), constants.SecretTLSKeyfile, "") + createTLSSNISecret(t, deployment.GetKubeCli(), "sni2", deployment.Namespace(), constants.SecretTLSKeyfile, "") + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Coordinators: api.MemberStatusList{ + firstCoordinatorStatus, + }, + }, + Images: createTestImages(false), + } + testCase.createTestPodData(deployment, api.ServerGroupCoordinators, firstCoordinatorStatus) + }, + ExpectedEvent: "member coordinator is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupCoordinatorsString, firstCoordinatorStatus.ID), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForCoordinator(firstCoordinatorStatus.ID, true, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + }, + Resources: emptyResources, + ReadinessProbe: createTestReadinessProbe(true, ""), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultCoordinatorTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupCoordinatorsString + "-" + firstCoordinatorStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupCoordinatorsString, + false, ""), + }, + }, + }, + { + Name: "Pod SNI Mounts - Enterprise - 3.6.0", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: func() api.TLSSpec { + s := tlsSpec.DeepCopy() + + s.SNI = &api.TLSSNISpec{ + Mapping: map[string][]string{ + "sni1": { + "a", + "b", + }, + "sni2": { + "c", + "d", + }, + }} + + return *s + }(), + }, + }, + Resources: func(t *testing.T, deployment *Deployment) { + createTLSSNISecret(t, deployment.GetKubeCli(), "sni1", deployment.Namespace(), constants.SecretTLSKeyfile, "") + createTLSSNISecret(t, deployment.GetKubeCli(), "sni2", deployment.Namespace(), constants.SecretTLSKeyfile, "") + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Coordinators: api.MemberStatusList{ + firstCoordinatorStatus, + }, + }, + Images: createTestImagesWithVersion(true, "3.6.0"), + } + testCase.createTestPodData(deployment, api.ServerGroupCoordinators, firstCoordinatorStatus) + }, + ExpectedEvent: "member coordinator is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupCoordinatorsString, firstCoordinatorStatus.ID), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForCoordinator(firstCoordinatorStatus.ID, true, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + }, + Resources: emptyResources, + ReadinessProbe: createTestReadinessProbe(true, ""), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultCoordinatorTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupCoordinatorsString + "-" + firstCoordinatorStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupCoordinatorsString, + false, ""), + }, + }, + }, + { + Name: "Pod SNI Mounts - 3.7.0", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: func() api.TLSSpec { + s := tlsSpec.DeepCopy() + + s.SNI = &api.TLSSNISpec{ + Mapping: map[string][]string{ + "sni1": { + "a", + "b", + }, + "sni2": { + "c", + "d", + }, + }} + + return *s + }(), + }, + }, + Resources: func(t *testing.T, deployment *Deployment) { + createTLSSNISecret(t, deployment.GetKubeCli(), "sni1", deployment.Namespace(), constants.SecretTLSKeyfile, "") + createTLSSNISecret(t, deployment.GetKubeCli(), "sni2", deployment.Namespace(), constants.SecretTLSKeyfile, "") + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Coordinators: api.MemberStatusList{ + firstCoordinatorStatus, + }, + }, + Images: createTestImagesWithVersion(false, "3.7.0"), + } + testCase.createTestPodData(deployment, api.ServerGroupCoordinators, firstCoordinatorStatus) + }, + ExpectedEvent: "member coordinator is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupCoordinatorsString, firstCoordinatorStatus.ID), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: createTestCommandForCoordinator(firstCoordinatorStatus.ID, true, false, false), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + }, + Resources: emptyResources, + ReadinessProbe: createTestReadinessProbe(true, ""), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultCoordinatorTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupCoordinatorsString + "-" + firstCoordinatorStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupCoordinatorsString, + false, ""), + }, + }, + }, + { + Name: "Pod SNI Mounts - Enterprise- 3.7.0", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: func() api.TLSSpec { + s := tlsSpec.DeepCopy() + + s.SNI = &api.TLSSNISpec{ + Mapping: map[string][]string{ + "sni1": { + "a", + "b", + }, + "sni2": { + "c", + "d", + }, + }} + + return *s + }(), + }, + }, + Resources: func(t *testing.T, deployment *Deployment) { + createTLSSNISecret(t, deployment.GetKubeCli(), "sni1", deployment.Namespace(), constants.SecretTLSKeyfile, "") + createTLSSNISecret(t, deployment.GetKubeCli(), "sni2", deployment.Namespace(), constants.SecretTLSKeyfile, "") + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + Coordinators: api.MemberStatusList{ + firstCoordinatorStatus, + }, + }, + Images: createTestImagesWithVersion(true, "3.7.0"), + } + testCase.createTestPodData(deployment, api.ServerGroupCoordinators, firstCoordinatorStatus) + }, + ExpectedEvent: "member coordinator is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupCoordinatorsString, firstCoordinatorStatus.ID), + { + Name: "sni-1b43a8b9b6df3d38b4ef394346283cd5aeda46a9b61d52da", + VolumeSource: core.VolumeSource{ + Secret: &core.SecretVolumeSource{ + SecretName: "sni1", + }, + }, + }, + { + Name: "sni-bbd5fc9d5151a1294ffb5de7b85ee74b7f4620021b5891e4", + VolumeSource: core.VolumeSource{ + Secret: &core.SecretVolumeSource{ + SecretName: "sni2", + }, + }, + }, + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: func() []string { + args := createTestCommandForCoordinator(firstCoordinatorStatus.ID, true, false, false) + args = append(args, fmt.Sprintf("--ssl.server-name-indication=a=%s/sni1/tls.keyfile", k8sutil.TLSSNIKeyfileVolumeMountDir), + fmt.Sprintf("--ssl.server-name-indication=b=%s/sni1/tls.keyfile", k8sutil.TLSSNIKeyfileVolumeMountDir), + fmt.Sprintf("--ssl.server-name-indication=c=%s/sni2/tls.keyfile", k8sutil.TLSSNIKeyfileVolumeMountDir), + fmt.Sprintf("--ssl.server-name-indication=d=%s/sni2/tls.keyfile", k8sutil.TLSSNIKeyfileVolumeMountDir)) + return args + }(), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + { + Name: "sni-1b43a8b9b6df3d38b4ef394346283cd5aeda46a9b61d52da", + MountPath: k8sutil.TLSSNIKeyfileVolumeMountDir + "/sni1", + ReadOnly: true, + }, + { + Name: "sni-bbd5fc9d5151a1294ffb5de7b85ee74b7f4620021b5891e4", + MountPath: k8sutil.TLSSNIKeyfileVolumeMountDir + "/sni2", + ReadOnly: true, + }, + }, + Resources: emptyResources, + ReadinessProbe: createTestReadinessProbe(true, ""), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultCoordinatorTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupCoordinatorsString + "-" + firstCoordinatorStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupCoordinatorsString, + false, ""), + }, + }, + }, + { + Name: "Pod SNI Mounts - Enterprise - 3.7.0 - DBServer", + ArangoDeployment: &api.ArangoDeployment{ + Spec: api.DeploymentSpec{ + Image: util.NewString(testImage), + Authentication: noAuthentication, + TLS: func() api.TLSSpec { + s := tlsSpec.DeepCopy() + + s.SNI = &api.TLSSNISpec{ + Mapping: map[string][]string{ + "sni1": { + "a", + "b", + }, + "sni2": { + "c", + "d", + }, + }} + + return *s + }(), + }, + }, + Resources: func(t *testing.T, deployment *Deployment) { + createTLSSNISecret(t, deployment.GetKubeCli(), "sni1", deployment.Namespace(), constants.SecretTLSKeyfile, "") + createTLSSNISecret(t, deployment.GetKubeCli(), "sni2", deployment.Namespace(), constants.SecretTLSKeyfile, "") + }, + Helper: func(t *testing.T, deployment *Deployment, testCase *testCaseStruct) { + deployment.status.last = api.DeploymentStatus{ + Members: api.DeploymentStatusMembers{ + DBServers: api.MemberStatusList{ + firstDBServerStatus, + }, + }, + Images: createTestImagesWithVersion(true, "3.7.0"), + } + testCase.createTestPodData(deployment, api.ServerGroupDBServers, firstDBServerStatus) + }, + ExpectedEvent: "member dbserver is created", + ExpectedPod: core.Pod{ + Spec: core.PodSpec{ + Volumes: []core.Volume{ + k8sutil.CreateVolumeEmptyDir(k8sutil.ArangodVolumeName), + createTestTLSVolume(api.ServerGroupDBServersString, firstDBServerStatus.ID), + }, + Containers: []core.Container{ + { + Name: k8sutil.ServerContainerName, + Image: testImage, + Command: func() []string { + args := createTestCommandForDBServer(firstDBServerStatus.ID, true, false, false) + return args + }(), + Ports: createTestPorts(), + VolumeMounts: []core.VolumeMount{ + k8sutil.ArangodVolumeMount(), + k8sutil.TlsKeyfileVolumeMount(), + }, + Resources: emptyResources, + LivenessProbe: createTestLivenessProbe(true, "", k8sutil.ArangoPort), + ImagePullPolicy: core.PullIfNotPresent, + SecurityContext: securityContext.NewSecurityContext(), + }, + }, + RestartPolicy: core.RestartPolicyNever, + TerminationGracePeriodSeconds: &defaultDBServerTerminationTimeout, + Hostname: testDeploymentName + "-" + api.ServerGroupDBServersString + "-" + firstDBServerStatus.ID, + Subdomain: testDeploymentName + "-int", + Affinity: k8sutil.CreateAffinity(testDeploymentName, api.ServerGroupDBServersString, + false, ""), + }, + }, + }, + } + + runTestCases(t, testCases...) +} diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index eab0fc2f1..adb146440 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -38,8 +38,13 @@ import ( ) func runTestCases(t *testing.T, testCases ...testCaseStruct) { - for _, testCase := range testCases { - runTestCase(t, testCase) + // This esure idempotency in generated outputs + for i := 0; i < 25; i++ { + t.Run(fmt.Sprintf("Iteration %d", i), func(t *testing.T) { + for _, testCase := range testCases { + runTestCase(t, testCase) + } + }) } } @@ -59,6 +64,10 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { _, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(testNamespace).Create(d.apiObject) require.NoError(t, err) + if testCase.Resources != nil { + testCase.Resources(t, d) + } + // Act err = d.resources.EnsurePods() diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index 1178dc2db..d22dbeb33 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -28,6 +28,8 @@ import ( "os" "testing" + "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver/jwt" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/resources" @@ -65,6 +67,7 @@ type testCaseStruct struct { Name string ArangoDeployment *api.ArangoDeployment Helper func(*testing.T, *Deployment, *testCaseStruct) + Resources func(*testing.T, *Deployment) config Config CompareChecksum *bool ExpectedError error @@ -391,17 +394,21 @@ func createTestPorts() []core.ContainerPort { } } -func createTestImages(enterprise bool) api.ImageInfoList { +func createTestImagesWithVersion(enterprise bool, version driver.Version) api.ImageInfoList { return api.ImageInfoList{ { Image: testImage, - ArangoDBVersion: testVersion, + ArangoDBVersion: version, ImageID: testImage, Enterprise: enterprise, }, } } +func createTestImages(enterprise bool) api.ImageInfoList { + return createTestImagesWithVersion(enterprise, testVersion) +} + func createTestExporterPorts(port uint16) []core.ContainerPort { return []core.ContainerPort{ { diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 35eb898bd..489f7ed30 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -372,3 +372,7 @@ func (i *ImageUpdatePod) GetNodeAffinity() *core.NodeAffinity { return pod.ReturnNodeAffinityOrNil(a) } + +func (i *ImageUpdatePod) Validate(secrets k8sutil.SecretInterface) error { + return nil +} diff --git a/pkg/deployment/pod/args.go b/pkg/deployment/pod/builder.go similarity index 72% rename from pkg/deployment/pod/args.go rename to pkg/deployment/pod/builder.go index 458175610..64f906238 100644 --- a/pkg/deployment/pod/args.go +++ b/pkg/deployment/pod/builder.go @@ -25,16 +25,25 @@ package pod import ( "github.com/arangodb/go-driver" deploymentApi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) type Input struct { + ApiObject meta.Object Deployment deploymentApi.DeploymentSpec + Status deploymentApi.DeploymentStatus GroupSpec deploymentApi.ServerGroupSpec Group deploymentApi.ServerGroup Version driver.Version + Enterprise bool AutoUpgrade bool + ID string } -type ArgumentsBuilder interface { - Create(i Input) []OptionPair +type Builder interface { + Args(i Input) k8sutil.OptionPairs + Volumes(i Input) ([]core.Volume, []core.VolumeMount) + Verify(i Input, s k8sutil.SecretInterface) error } diff --git a/pkg/deployment/pod/sni.go b/pkg/deployment/pod/sni.go new file mode 100644 index 000000000..216f8723c --- /dev/null +++ b/pkg/deployment/pod/sni.go @@ -0,0 +1,146 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package pod + +import ( + "crypto/sha256" + "fmt" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func GroupSNISupported(mode api.DeploymentMode, group api.ServerGroup) bool { + switch mode { + case api.DeploymentModeCluster: + return group == api.ServerGroupCoordinators + + case api.DeploymentModeSingle: + fallthrough + case api.DeploymentModeActiveFailover: + return group == api.ServerGroupSingle + default: + return false + } +} + +func SNI() Builder { + return sni{} +} + +type sni struct{} + +func (s sni) isSupported(i Input) bool { + if !i.Deployment.TLS.IsSecure() { + return false + } + + if i.Version.CompareTo("3.7.0") < 0 || !i.Enterprise { + // We need 3.7.0+ and Enterprise to support this + return false + } + + return GroupSNISupported(i.Deployment.Mode.Get(), i.Group) +} + +func (s sni) Verify(i Input, secrets k8sutil.SecretInterface) error { + if !s.isSupported(i) { + return nil + } + + for _, secret := range util.SortKeys(i.Deployment.TLS.GetTLSSNISpec().Mapping) { + kubeSecret, err := secrets.Get(secret, meta.GetOptions{}) + if err != nil { + return err + } + + _, ok := kubeSecret.Data[constants.SecretTLSKeyfile] + if !ok { + return errors.Errorf("Unable to find secret key %s/%s for SNI", secret, constants.SecretTLSKeyfile) + } + } + return nil +} + +func (s sni) Volumes(i Input) ([]core.Volume, []core.VolumeMount) { + if !s.isSupported(i) { + return nil, nil + } + + sni := i.Deployment.TLS.GetTLSSNISpec() + volumes := make([]core.Volume, 0, len(sni.Mapping)) + volumeMounts := make([]core.VolumeMount, 0, len(sni.Mapping)) + + for _, secret := range util.SortKeys(sni.Mapping) { + secretNameSha := fmt.Sprintf("%0x", sha256.Sum256([]byte(secret))) + + secretNameSha = fmt.Sprintf("sni-%s", secretNameSha[:48]) + + vol := core.Volume{ + Name: secretNameSha, + VolumeSource: core.VolumeSource{ + Secret: &core.SecretVolumeSource{ + SecretName: secret, + }, + }, + } + + volMount := core.VolumeMount{ + Name: secretNameSha, + MountPath: fmt.Sprintf("%s/%s", k8sutil.TLSSNIKeyfileVolumeMountDir, secret), + ReadOnly: true, + } + + volumes = append(volumes, vol) + volumeMounts = append(volumeMounts, volMount) + } + + return volumes, volumeMounts +} + +func (s sni) Args(i Input) k8sutil.OptionPairs { + if !s.isSupported(i) { + return nil + } + + opts := k8sutil.CreateOptionPairs() + + for _, volume := range util.SortKeys(i.Deployment.TLS.GetTLSSNISpec().Mapping) { + servers, ok := i.Deployment.TLS.SNI.Mapping[volume] + if !ok { + continue + } + + for _, server := range servers { + opts.Addf("--ssl.server-name-indication", "%s=%s/%s/%s", server, k8sutil.TLSSNIKeyfileVolumeMountDir, volume, constants.SecretTLSKeyfile) + } + } + + return opts +} diff --git a/pkg/deployment/pod/upgrade.go b/pkg/deployment/pod/upgrade.go index 44bcca740..562eb2d7c 100644 --- a/pkg/deployment/pod/upgrade.go +++ b/pkg/deployment/pod/upgrade.go @@ -22,25 +22,37 @@ package pod -import deploymentApi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" +import ( + deploymentApi "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + core "k8s.io/api/core/v1" +) -func AutoUpgrade() ArgumentsBuilder { - return autoUpgradeArgs{} +func AutoUpgrade() Builder { + return autoUpgrade{} } -type autoUpgradeArgs struct{} +type autoUpgrade struct{} -func (u autoUpgradeArgs) Create(i Input) []OptionPair { +func (u autoUpgrade) Verify(i Input, s k8sutil.SecretInterface) error { + return nil +} + +func (u autoUpgrade) Volumes(i Input) ([]core.Volume, []core.VolumeMount) { + return nil, nil +} + +func (u autoUpgrade) Args(i Input) k8sutil.OptionPairs { if !i.AutoUpgrade { - return NewOptionPair() + return nil } if i.Version.CompareTo("3.6.0") >= 0 { switch i.Group { case deploymentApi.ServerGroupCoordinators: - return NewOptionPair(OptionPair{"--cluster.upgrade", "online"}) + return k8sutil.NewOptionPair(k8sutil.OptionPair{"--cluster.upgrade", "online"}) } } - return NewOptionPair(OptionPair{"--database.auto-upgrade", "true"}) + return k8sutil.NewOptionPair(k8sutil.OptionPair{"--database.auto-upgrade", "true"}) } diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index ed4f2086e..d12c52bf8 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -96,6 +96,9 @@ type ActionContext interface { // GetImageInfo returns the image info for an image with given name. // Returns: (info, infoFound) GetImageInfo(imageName string) (api.ImageInfo, bool) + // GetImageInfo returns the image info for an current image. + // Returns: (info, infoFound) + GetCurrentImageInfo() (api.ImageInfo, bool) // SetCurrentImage changes the CurrentImage field in the deployment // status to the given image. SetCurrentImage(imageInfo api.ImageInfo) error @@ -113,6 +116,7 @@ type ActionContext interface { EnableScalingCluster() error // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error + SecretsInterface() k8sutil.SecretInterface } // newActionContext creates a new ActionContext implementation. @@ -129,6 +133,10 @@ type actionContext struct { context Context } +func (ac *actionContext) SecretsInterface() k8sutil.SecretInterface { + return ac.context.SecretsInterface() +} + func (ac *actionContext) GetShardSyncStatus() bool { return ac.context.GetShardSyncStatus() } @@ -342,6 +350,18 @@ func (ac *actionContext) GetImageInfo(imageName string) (api.ImageInfo, bool) { return status.Images.GetByImage(imageName) } +// GetImageInfo returns the image info for an current image. +// Returns: (info, infoFound) +func (ac *actionContext) GetCurrentImageInfo() (api.ImageInfo, bool) { + status, _ := ac.context.GetStatus() + + if status.CurrentImage == nil { + return api.ImageInfo{}, false + } + + return *status.CurrentImage, true +} + // SetCurrentImage changes the CurrentImage field in the deployment // status to the given image. func (ac *actionContext) SetCurrentImage(imageInfo api.ImageInfo) error { diff --git a/pkg/deployment/reconcile/action_tls_sni_update.go b/pkg/deployment/reconcile/action_tls_sni_update.go new file mode 100644 index 000000000..768e07b2b --- /dev/null +++ b/pkg/deployment/reconcile/action_tls_sni_update.go @@ -0,0 +1,83 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func init() { + registerAction(api.ActionTypeUpdateTLSSNI, newTLSSNIUpdate) +} + +func newTLSSNIUpdate(log zerolog.Logger, action api.Action, actionCtx ActionContext) Action { + a := &tlsSNIUpdate{} + + a.actionImpl = newActionImplDefRef(log, action, actionCtx, tlsSNIUpdateTimeout) + + return a +} + +type tlsSNIUpdate struct { + actionImpl + + actionEmptyStart +} + +func (t *tlsSNIUpdate) CheckProgress(ctx context.Context) (bool, bool, error) { + spec := t.actionCtx.GetSpec() + if !spec.TLS.IsSecure() { + return true, false, nil + } + + if i, ok := t.actionCtx.GetCurrentImageInfo(); !ok || !i.Enterprise { + return true, false, nil + } + + sni := spec.TLS.SNI + if sni == nil { + return true, false, nil + } + + fetchedSecrets, err := mapTLSSNIConfig(t.log, *sni, t.actionCtx.SecretsInterface()) + if err != nil { + t.log.Warn().Err(err).Msg("Unable to get SNI desired state") + return true, false, nil + } + + c, err := t.actionCtx.GetServerClient(ctx, t.action.Group, t.action.MemberID) + if err != nil { + t.log.Warn().Err(err).Msg("Unable to get client") + return true, false, nil + } + + if ok, err := compareTLSSNIConfig(ctx, c.Connection(), fetchedSecrets, true); err != nil { + t.log.Warn().Err(err).Msg("Unable to compare TLS config") + return true, false, nil + } else { + return ok, false, nil + } +} diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index 11cfffbd2..bb715831c 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -28,11 +28,9 @@ import ( "github.com/arangodb/arangosync-client/client" driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + v1 "k8s.io/api/core/v1" ) // Context provides methods to the reconcile package. @@ -97,9 +95,6 @@ type Context interface { // DeleteSecret removes the Secret with given name. // If the secret does not exist, the error is ignored. DeleteSecret(secretName string) error - // GetExpectedPodArguments creates command line arguments for a server in the given group with given ID. - GetExpectedPodArguments(apiObject metav1.Object, deplSpec api.DeploymentSpec, group api.ServerGroup, - agents api.MemberStatusList, id string, version driver.Version) []string // GetDeploymentHealth returns a copy of the latest known state of cluster health GetDeploymentHealth() (driver.ClusterHealth, error) // GetShardSyncStatus returns true if all shards are in sync @@ -118,4 +113,6 @@ type Context interface { SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + // SecretsInterface return secret interface + SecretsInterface() k8sutil.SecretInterface } diff --git a/pkg/deployment/reconcile/helper_tls_sni.go b/pkg/deployment/reconcile/helper_tls_sni.go new file mode 100644 index 000000000..d35de52cd --- /dev/null +++ b/pkg/deployment/reconcile/helper_tls_sni.go @@ -0,0 +1,101 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + "crypto/sha256" + "fmt" + + "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/tls" + "github.com/arangodb/kube-arangodb/pkg/util/constants" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" + "github.com/rs/zerolog" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func mapTLSSNIConfig(log zerolog.Logger, sni api.TLSSNISpec, secrets k8sutil.SecretInterface) (map[string]string, error) { + fetchedSecrets := map[string]string{} + + mapping := sni.Mapping + if len(mapping) == 0 { + return fetchedSecrets, nil + } + + for name, servers := range mapping { + secret, err := secrets.Get(name, meta.GetOptions{}) + if err != nil { + return nil, errors.WithMessage(err, "Unable to get SNI secret") + } + + tlsKey, ok := secret.Data[constants.SecretTLSKeyfile] + if !ok { + return nil, errors.Errorf("Not found tls keyfile key in SNI secret") + } + + tlsKeyChecksum := fmt.Sprintf("%0x", sha256.Sum256(tlsKey)) + + for _, server := range servers { + if _, ok := fetchedSecrets[server]; ok { + return nil, errors.Errorf("Not found tls key in SNI secret") + } + fetchedSecrets[server] = tlsKeyChecksum + } + } + + return fetchedSecrets, nil +} + +func compareTLSSNIConfig(ctx context.Context, c driver.Connection, m map[string]string, refresh bool) (bool, error) { + tlsClient := tls.NewClient(c) + + f := tlsClient.GetTLS + if refresh { + f = tlsClient.RefreshTLS + } + + tlsDetails, err := f(ctx) + if err != nil { + return false, errors.WithMessage(err, "Unable to fetch TLS SNI state") + } + + if len(m) != len(tlsDetails.Result.SNI) { + return false, errors.Errorf("Count of SNI mounted secrets does not match") + } + + for key, value := range tlsDetails.Result.SNI { + currentValue, ok := m[key] + if !ok { + return false, errors.Errorf("Unable to fetch TLS SNI state") + } + + if value.Checksum != currentValue { + return false, nil + } + } + + return true, nil +} diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index f122f1ad4..54f4fe93f 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -27,6 +27,8 @@ import ( "fmt" "time" + "golang.org/x/net/context" + "github.com/arangodb/kube-arangodb/pkg/deployment/agency" driver "github.com/arangodb/go-driver" @@ -52,7 +54,7 @@ type upgradeDecision struct { // CreatePlan considers the current specification & status of the deployment creates a plan to // get the status in line with the specification. // If a plan already exists, nothing is done. -func (d *Reconciler) CreatePlan() (error, bool) { +func (d *Reconciler) CreatePlan(ctx context.Context) (error, bool) { // Get all current pods pods, err := d.context.GetOwnedPods() if err != nil { @@ -64,8 +66,8 @@ func (d *Reconciler) CreatePlan() (error, bool) { apiObject := d.context.GetAPIObject() spec := d.context.GetSpec() status, lastVersion := d.context.GetStatus() - ctx := newPlanBuilderContext(d.context) - newPlan, changed := createPlan(d.log, apiObject, status.Plan, spec, status, pods, ctx) + builderCtx := newPlanBuilderContext(d.context) + newPlan, changed := createPlan(ctx, d.log, apiObject, status.Plan, spec, status, pods, builderCtx) // If not change, we're done if !changed { @@ -84,13 +86,13 @@ func (d *Reconciler) CreatePlan() (error, bool) { return nil, true } -func fetchAgency(log zerolog.Logger, +func fetchAgency(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) (*agency.ArangoPlanDatabases, error) { if spec.GetMode() != api.DeploymentModeCluster && spec.GetMode() != api.DeploymentModeActiveFailover { return nil, nil } else if status.Members.Agents.MembersReady() > 0 { - agencyCtx, agencyCancel := goContext.WithTimeout(goContext.Background(), time.Minute) + agencyCtx, agencyCancel := goContext.WithTimeout(ctx, time.Minute) defer agencyCancel() ret := &agency.ArangoPlanDatabases{} @@ -108,10 +110,10 @@ func fetchAgency(log zerolog.Logger, // createPlan considers the given specification & status and creates a plan to get the status in line with the specification. // If a plan already exists, the given plan is returned with false. // Otherwise the new plan is returned with a boolean true. -func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, +func createPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, currentPlan api.Plan, spec api.DeploymentSpec, status api.DeploymentStatus, pods []v1.Pod, - context PlanBuilderContext) (api.Plan, bool) { + builderCtx PlanBuilderContext) (api.Plan, bool) { if !currentPlan.IsEmpty() { // Plan already exists, complete that first @@ -119,7 +121,7 @@ func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, } // Fetch agency plan - agencyPlan, agencyErr := fetchAgency(log, spec, status, context) + agencyPlan, agencyErr := fetchAgency(ctx, log, spec, status, builderCtx) // Check for various scenario's var plan api.Plan @@ -202,7 +204,7 @@ func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, // Check for the need to rotate one or more members if plan.IsEmpty() { - newPlan, idle := createRotateOrUpgradePlan(log, apiObject, spec, status, context, pods) + newPlan, idle := createRotateOrUpgradePlan(log, apiObject, spec, status, builderCtx, pods) if idle { plan = append(plan, api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")) @@ -213,17 +215,21 @@ func createPlan(log zerolog.Logger, apiObject k8sutil.APIObject, // Check for the need to rotate TLS certificate of a members if plan.IsEmpty() { - plan = createRotateTLSServerCertificatePlan(log, spec, status, context.GetTLSKeyfile) + plan = createRotateTLSServerCertificatePlan(log, spec, status, builderCtx.GetTLSKeyfile) } // Check for changes storage classes or requirements if plan.IsEmpty() { - plan = createRotateServerStoragePlan(log, apiObject, spec, status, context.GetPvc, context.CreateEvent) + plan = createRotateServerStoragePlan(log, apiObject, spec, status, builderCtx.GetPvc, builderCtx.CreateEvent) } // Check for the need to rotate TLS CA certificate and all members if plan.IsEmpty() { - plan = createRotateTLSCAPlan(log, apiObject, spec, status, context.GetTLSCA, context.CreateEvent) + plan = createRotateTLSCAPlan(log, apiObject, spec, status, builderCtx.GetTLSCA, builderCtx.CreateEvent) + } + + if plan.IsEmpty() { + plan = createRotateTLSServerSNIPlan(ctx, log, spec, status, builderCtx) } // Return plan diff --git a/pkg/deployment/reconcile/plan_builder_context.go b/pkg/deployment/reconcile/plan_builder_context.go index d19447ec4..2bba83f1e 100644 --- a/pkg/deployment/reconcile/plan_builder_context.go +++ b/pkg/deployment/reconcile/plan_builder_context.go @@ -25,11 +25,11 @@ package reconcile import ( "context" - driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" core "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // PlanBuilderContext contains context methods provided to plan builders. @@ -45,9 +45,6 @@ type PlanBuilderContext interface { CreateEvent(evt *k8sutil.Event) // GetPvc gets a PVC by the given name, in the samespace of the deployment. GetPvc(pvcName string) (*core.PersistentVolumeClaim, error) - // GetExpectedPodArguments creates command line arguments for a server in the given group with given ID. - GetExpectedPodArguments(apiObject metav1.Object, deplSpec api.DeploymentSpec, group api.ServerGroup, - agents api.MemberStatusList, id string, version driver.Version) []string // GetShardSyncStatus returns true if all shards are in sync GetShardSyncStatus() bool // InvalidateSyncStatus resets the sync state to false and triggers an inspection @@ -60,6 +57,10 @@ type PlanBuilderContext interface { RenderPodForMember(spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) // SelectImage select currently used image by pod SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) + // GetServerClient returns a cached client for a specific server. + GetServerClient(ctx context.Context, group api.ServerGroup, id string) (driver.Client, error) + // SecretsInterface return secret interface + SecretsInterface() k8sutil.SecretInterface } // newPlanBuilderContext creates a PlanBuilderContext from the given context diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index e8a311f91..535d0faa7 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -55,6 +55,10 @@ type testContext struct { RecordedEvent *k8sutil.Event } +func (c *testContext) SecretsInterface() k8sutil.SecretInterface { + panic("implement me") +} + func (c *testContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { panic("implement me") } @@ -226,6 +230,9 @@ func addAgentsToStatus(t *testing.T, status *api.DeploymentStatus, count int) { // TestCreatePlanSingleScale creates a `single` deployment to test the creating of scaling plan. func TestCreatePlanSingleScale(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := &testContext{} log := zerolog.Nop() spec := api.DeploymentSpec{ @@ -242,7 +249,7 @@ func TestCreatePlanSingleScale(t *testing.T) { // Test with empty status var status api.DeploymentStatus - newPlan, changed := createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) assert.Len(t, newPlan, 0) // Single mode does not scale @@ -253,7 +260,7 @@ func TestCreatePlanSingleScale(t *testing.T) { PodName: "something", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed = createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) assert.Len(t, newPlan, 0) // Single mode does not scale @@ -268,13 +275,16 @@ func TestCreatePlanSingleScale(t *testing.T) { PodName: "something1", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed = createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) assert.Len(t, newPlan, 0) // Single mode does not scale } // TestCreatePlanActiveFailoverScale creates a `ActiveFailover` deployment to test the creating of scaling plan. func TestCreatePlanActiveFailoverScale(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := &testContext{} log := zerolog.Nop() spec := api.DeploymentSpec{ @@ -294,7 +304,7 @@ func TestCreatePlanActiveFailoverScale(t *testing.T) { var status api.DeploymentStatus addAgentsToStatus(t, &status, 3) - newPlan, changed := createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) require.Len(t, newPlan, 2) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -307,7 +317,7 @@ func TestCreatePlanActiveFailoverScale(t *testing.T) { PodName: "something", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed = createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) require.Len(t, newPlan, 1) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -332,7 +342,7 @@ func TestCreatePlanActiveFailoverScale(t *testing.T) { PodName: "something4", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed = createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) require.Len(t, newPlan, 2) // Note: Downscaling is only down 1 at a time assert.Equal(t, api.ActionTypeShutdownMember, newPlan[0].Type) @@ -343,6 +353,9 @@ func TestCreatePlanActiveFailoverScale(t *testing.T) { // TestCreatePlanClusterScale creates a `cluster` deployment to test the creating of scaling plan. func TestCreatePlanClusterScale(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + c := &testContext{} log := zerolog.Nop() spec := api.DeploymentSpec{ @@ -361,7 +374,7 @@ func TestCreatePlanClusterScale(t *testing.T) { var status api.DeploymentStatus addAgentsToStatus(t, &status, 3) - newPlan, changed := createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed := createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) require.Len(t, newPlan, 6) // Adding 3 dbservers & 3 coordinators (note: agents do not scale now) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -394,7 +407,7 @@ func TestCreatePlanClusterScale(t *testing.T) { PodName: "coordinator1", }, } - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed = createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) require.Len(t, newPlan, 3) assert.Equal(t, api.ActionTypeAddMember, newPlan[0].Type) @@ -431,7 +444,7 @@ func TestCreatePlanClusterScale(t *testing.T) { } spec.DBServers.Count = util.NewInt(1) spec.Coordinators.Count = util.NewInt(1) - newPlan, changed = createPlan(log, depl, nil, spec, status, nil, c) + newPlan, changed = createPlan(ctx, log, depl, nil, spec, status, nil, c) assert.True(t, changed) require.Len(t, newPlan, 5) // Note: Downscaling is done 1 at a time assert.Equal(t, api.ActionTypeCleanOutMember, newPlan[0].Type) @@ -768,6 +781,9 @@ func TestCreatePlan(t *testing.T) { //nolint:scopelint t.Run(testCase.Name, func(t *testing.T) { // Arrange + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + h := &LastLogRecord{} logger := zerolog.New(ioutil.Discard).Hook(h) r := NewReconciler(logger, testCase.context) @@ -776,7 +792,7 @@ func TestCreatePlan(t *testing.T) { if testCase.Helper != nil { testCase.Helper(testCase.context.ArangoDeployment) } - err, _ := r.CreatePlan() + err, _ := r.CreatePlan(ctx) // Assert if testCase.ExpectedEvent != nil { diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go new file mode 100644 index 000000000..eff93082e --- /dev/null +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -0,0 +1,101 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package reconcile + +import ( + "context" + + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/rs/zerolog" +) + +func createRotateTLSServerSNIPlan(ctx context.Context, log zerolog.Logger, spec api.DeploymentSpec, status api.DeploymentStatus, context PlanBuilderContext) api.Plan { + if !spec.TLS.IsSecure() { + return nil + } + + if i := status.CurrentImage; i == nil || !i.Enterprise { + return nil + } + + sni := spec.TLS.SNI + if sni == nil { + return nil + } + + fetchedSecrets, err := mapTLSSNIConfig(log, *sni, context.SecretsInterface()) + if err != nil { + log.Warn().Err(err).Msg("Unable to get SNI desired state") + return nil + } + + var plan api.Plan + status.Members.ForeachServerGroup(func(group api.ServerGroup, members api.MemberStatusList) error { + if !pod.GroupSNISupported(spec.Mode.Get(), group) { + return nil + } + + for _, m := range members { + if !plan.IsEmpty() { + // Only 1 member at a time + return nil + } + + if m.Phase != api.MemberPhaseCreated { + // Only make changes when phase is created + continue + } + + if m.ArangoVersion.CompareTo("3.7.0") < 0 { + continue + } + + c, err := context.GetServerClient(ctx, group, m.ID) + if err != nil { + log.Warn().Err(err).Msg("Unable to get client") + continue + } + + if ok, err := compareTLSSNIConfig(ctx, c.Connection(), fetchedSecrets, false); err != nil { + log.Warn().Err(err).Msg("SNI compare failed") + return nil + + } else if !ok { + switch sni.Mode.Get() { + case api.TLSSNIRotateModeRecreate: + plan = append(plan, createRotateMemberPlan(log, m, group, "SNI Secret needs update")...) + case api.TLSSNIRotateModeInPlace: + plan = append(plan, + api.NewAction(api.ActionTypeUpdateTLSSNI, group, m.ID, "SNI Secret needs update")) + default: + log.Warn().Msg("SNI mode rotation is unknown") + continue + } + } + } + return nil + }) + return plan +} diff --git a/pkg/deployment/reconcile/timeouts.go b/pkg/deployment/reconcile/timeouts.go index adc35ab95..7462c7c8b 100644 --- a/pkg/deployment/reconcile/timeouts.go +++ b/pkg/deployment/reconcile/timeouts.go @@ -37,7 +37,7 @@ const ( shutdownMemberTimeout = time.Minute * 30 upgradeMemberTimeout = time.Hour * 6 waitForMemberUpTimeout = time.Minute * 15 - upToDateUpdateTimeout = time.Minute + tlsSNIUpdateTimeout = time.Minute * 10 shutdownTimeout = time.Second * 15 ) diff --git a/pkg/deployment/resources/exporter.go b/pkg/deployment/resources/exporter.go index 60d909c35..28ef81f82 100644 --- a/pkg/deployment/resources/exporter.go +++ b/pkg/deployment/resources/exporter.go @@ -28,8 +28,6 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/arangodb/kube-arangodb/pkg/deployment/pod" - "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" v1 "k8s.io/api/core/v1" @@ -65,25 +63,25 @@ func ArangodbExporterContainer(image string, args []string, livenessProbe *k8sut func createExporterArgs(spec api.DeploymentSpec) []string { tokenpath := filepath.Join(k8sutil.ExporterJWTVolumeMountDir, constants.SecretKeyToken) - options := make([]pod.OptionPair, 0, 64) + options := make([]k8sutil.OptionPair, 0, 64) scheme := "http" if spec.IsSecure() { scheme = "https" } options = append(options, - pod.OptionPair{"--arangodb.jwt-file", tokenpath}, - pod.OptionPair{"--arangodb.endpoint", scheme + "://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, + k8sutil.OptionPair{"--arangodb.jwt-file", tokenpath}, + k8sutil.OptionPair{"--arangodb.endpoint", scheme + "://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, ) keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) if spec.IsSecure() { options = append(options, - pod.OptionPair{"--ssl.keyfile", keyPath}, + k8sutil.OptionPair{"--ssl.keyfile", keyPath}, ) } if port := spec.Metrics.GetPort(); port != k8sutil.ArangoExporterPort { options = append(options, - pod.OptionPair{"--server.address", fmt.Sprintf(":%d", port)}, + k8sutil.OptionPair{"--server.address", fmt.Sprintf(":%d", port)}, ) } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 2820c9475..d1c349c0a 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -29,7 +29,6 @@ import ( "net" "net/url" "path/filepath" - "sort" "strconv" "sync" "time" @@ -67,180 +66,124 @@ func versionHasJWTSecretKeyfile(v driver.Version) bool { } // createArangodArgs creates command line arguments for an arangod server in the given group. -func createArangodArgs(apiObject metav1.Object, deplSpec api.DeploymentSpec, group api.ServerGroup, - agents api.MemberStatusList, id string, version driver.Version, autoUpgrade bool) []string { - options := make([]pod.OptionPair, 0, 64) - svrSpec := deplSpec.GetServerGroupSpec(group) - - i := pod.Input{ - Deployment: deplSpec, - Group: group, - GroupSpec: svrSpec, - Version: version, - AutoUpgrade: autoUpgrade, - } +func createArangodArgs(input pod.Input) []string { + options := k8sutil.CreateOptionPairs(64) //scheme := NewURLSchemes(bsCfg.SslKeyFile != "").Arangod scheme := "tcp" - if deplSpec.IsSecure() { + if input.Deployment.IsSecure() { scheme = "ssl" } - options = append(options, - pod.OptionPair{"--server.endpoint", fmt.Sprintf("%s://%s:%d", scheme, deplSpec.GetListenAddr(), k8sutil.ArangoPort)}, - ) + + options.Addf("--server.endpoint", "%s://%s:%d", scheme, input.Deployment.GetListenAddr(), k8sutil.ArangoPort) // Authentication - if deplSpec.IsAuthenticated() { + if input.Deployment.IsAuthenticated() { // With authentication - options = append(options, - pod.OptionPair{"--server.authentication", "true"}, - ) - if versionHasJWTSecretKeyfile(version) { + options.Add("--server.authentication", "true") + + if versionHasJWTSecretKeyfile(input.Version) { keyPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) - options = append(options, - pod.OptionPair{"--server.jwt-secret-keyfile", keyPath}, - ) + options.Add("--server.jwt-secret-keyfile", keyPath) } else { - options = append(options, - pod.OptionPair{"--server.jwt-secret", "$(" + constants.EnvArangodJWTSecret + ")"}, - ) + options.Addf("--server.jwt-secret", "$(%s)", constants.EnvArangodJWTSecret) } } else { // Without authentication - options = append(options, - pod.OptionPair{"--server.authentication", "false"}, - ) + options.Add("--server.authentication", "false") } // Storage engine - options = append(options, - pod.OptionPair{"--server.storage-engine", deplSpec.GetStorageEngine().AsArangoArgument()}, - ) + options.Add("--server.storage-engine", input.Deployment.GetStorageEngine().AsArangoArgument()) // Logging - options = append(options, - pod.OptionPair{"--log.level", "INFO"}, - ) + options.Add("--log.level", "INFO") // TLS - if deplSpec.IsSecure() { + if input.Deployment.IsSecure() { keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) - options = append(options, - pod.OptionPair{"--ssl.keyfile", keyPath}, - pod.OptionPair{"--ssl.ecdh-curve", ""}, // This way arangod accepts curves other than P256 as well. - ) - /*if bsCfg.SslKeyFile != "" { - if bsCfg.SslCAFile != "" { - sslSection.Settings["cafile"] = bsCfg.SslCAFile - } - config = append(config, sslSection) - }*/ + options.Add("--ssl.keyfile", keyPath) + options.Add("--ssl.ecdh-curve", "") // This way arangod accepts curves other than P256 as well. } // RocksDB - if deplSpec.RocksDB.IsEncrypted() { + if input.Deployment.RocksDB.IsEncrypted() { keyPath := filepath.Join(k8sutil.RocksDBEncryptionVolumeMountDir, constants.SecretEncryptionKey) - options = append(options, - pod.OptionPair{"--rocksdb.encryption-keyfile", keyPath}, - ) + options.Add("--rocksdb.encryption-keyfile", keyPath) } - options = append(options, - pod.OptionPair{"--database.directory", k8sutil.ArangodVolumeMountDir}, - pod.OptionPair{"--log.output", "+"}, - ) + options.Add("--database.directory", k8sutil.ArangodVolumeMountDir) + options.Add("--log.output", "+") - options = append(options, pod.AutoUpgrade().Create(i)...) + options.Merge(pod.AutoUpgrade().Args(input)) + options.Merge(pod.SNI().Args(input)) - versionHasAdvertisedEndpoint := versionHasAdvertisedEndpoint(version) + versionHasAdvertisedEndpoint := versionHasAdvertisedEndpoint(input.Version) /* if config.ServerThreads != 0 { options = append(options, - pod.OptionPair{"--server.threads", strconv.Itoa(config.ServerThreads)}) + k8sutil.OptionPair{"--server.threads", strconv.Itoa(config.ServerThreads)}) }*/ /*if config.DebugCluster { options = append(options, - pod.OptionPair{"--log.level", "startup=trace"}) + k8sutil.OptionPair{"--log.level", "startup=trace"}) }*/ - myTCPURL := scheme + "://" + net.JoinHostPort(k8sutil.CreatePodDNSName(apiObject, group.AsRole(), id), strconv.Itoa(k8sutil.ArangoPort)) + myTCPURL := scheme + "://" + net.JoinHostPort(k8sutil.CreatePodDNSName(input.ApiObject, input.Group.AsRole(), input.ID), strconv.Itoa(k8sutil.ArangoPort)) addAgentEndpoints := false - switch group { + switch input.Group { case api.ServerGroupAgents: - options = append(options, - pod.OptionPair{"--agency.disaster-recovery-id", id}, - pod.OptionPair{"--agency.activate", "true"}, - pod.OptionPair{"--agency.my-address", myTCPURL}, - pod.OptionPair{"--agency.size", strconv.Itoa(deplSpec.Agents.GetCount())}, - pod.OptionPair{"--agency.supervision", "true"}, - pod.OptionPair{"--foxx.queues", "false"}, - pod.OptionPair{"--server.statistics", "false"}, - ) - for _, p := range agents { - if p.ID != id { - dnsName := k8sutil.CreatePodDNSName(apiObject, api.ServerGroupAgents.AsRole(), p.ID) - options = append(options, - pod.OptionPair{"--agency.endpoint", fmt.Sprintf("%s://%s", scheme, net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort)))}, - ) + options.Add("--agency.disaster-recovery-id", input.ID) + options.Add("--agency.activate", "true") + options.Add("--agency.my-address", myTCPURL) + options.Addf("--agency.size", "%d", input.Deployment.Agents.GetCount()) + options.Add("--agency.supervision", "true") + options.Add("--foxx.queues", "false") + options.Add("--server.statistics", "false") + for _, p := range input.Status.Members.Agents { + if p.ID != input.ID { + dnsName := k8sutil.CreatePodDNSName(input.ApiObject, api.ServerGroupAgents.AsRole(), p.ID) + options.Addf("--agency.endpoint", "%s://%s", scheme, net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))) } } case api.ServerGroupDBServers: addAgentEndpoints = true - options = append(options, - pod.OptionPair{"--cluster.my-address", myTCPURL}, - pod.OptionPair{"--cluster.my-role", "PRIMARY"}, - pod.OptionPair{"--foxx.queues", "false"}, - pod.OptionPair{"--server.statistics", "true"}, - ) + options.Add("--cluster.my-address", myTCPURL) + options.Add("--cluster.my-role", "PRIMARY") + options.Add("--foxx.queues", "false") + options.Add("--server.statistics", "true") case api.ServerGroupCoordinators: addAgentEndpoints = true - options = append(options, - pod.OptionPair{"--cluster.my-address", myTCPURL}, - pod.OptionPair{"--cluster.my-role", "COORDINATOR"}, - pod.OptionPair{"--foxx.queues", "true"}, - pod.OptionPair{"--server.statistics", "true"}, - ) - if deplSpec.ExternalAccess.HasAdvertisedEndpoint() && versionHasAdvertisedEndpoint { - options = append(options, - pod.OptionPair{"--cluster.my-advertised-endpoint", deplSpec.ExternalAccess.GetAdvertisedEndpoint()}, - ) + options.Add("--cluster.my-address", myTCPURL) + options.Add("--cluster.my-role", "COORDINATOR") + options.Add("--foxx.queues", "true") + options.Add("--server.statistics", "true") + if input.Deployment.ExternalAccess.HasAdvertisedEndpoint() && versionHasAdvertisedEndpoint { + options.Add("--cluster.my-advertised-endpoint", input.Deployment.ExternalAccess.GetAdvertisedEndpoint()) } case api.ServerGroupSingle: - options = append(options, - pod.OptionPair{"--foxx.queues", "true"}, - pod.OptionPair{"--server.statistics", "true"}, - ) - if deplSpec.GetMode() == api.DeploymentModeActiveFailover { + options.Add("--foxx.queues", "true") + options.Add("--server.statistics", "true") + if input.Deployment.GetMode() == api.DeploymentModeActiveFailover { addAgentEndpoints = true - options = append(options, - pod.OptionPair{"--replication.automatic-failover", "true"}, - pod.OptionPair{"--cluster.my-address", myTCPURL}, - pod.OptionPair{"--cluster.my-role", "SINGLE"}, - ) - if deplSpec.ExternalAccess.HasAdvertisedEndpoint() && versionHasAdvertisedEndpoint { - options = append(options, - pod.OptionPair{"--cluster.my-advertised-endpoint", deplSpec.ExternalAccess.GetAdvertisedEndpoint()}, - ) + options.Add("--replication.automatic-failover", "true") + options.Add("--cluster.my-address", myTCPURL) + options.Add("--cluster.my-role", "SINGLE") + if input.Deployment.ExternalAccess.HasAdvertisedEndpoint() && versionHasAdvertisedEndpoint { + options.Add("--cluster.my-advertised-endpoint", input.Deployment.ExternalAccess.GetAdvertisedEndpoint()) } } } if addAgentEndpoints { - for _, p := range agents { - dnsName := k8sutil.CreatePodDNSName(apiObject, api.ServerGroupAgents.AsRole(), p.ID) - options = append(options, - pod.OptionPair{"--cluster.agency-endpoint", - fmt.Sprintf("%s://%s", scheme, net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort)))}, - ) + for _, p := range input.Status.Members.Agents { + dnsName := k8sutil.CreatePodDNSName(input.ApiObject, api.ServerGroupAgents.AsRole(), p.ID) + options.Addf("--cluster.agency-endpoint", "%s://%s", scheme, net.JoinHostPort(dnsName, strconv.Itoa(k8sutil.ArangoPort))) } } - args := make([]string, 0, len(options)+len(svrSpec.Args)) - sort.Slice(options, func(i, j int) bool { - return options[i].CompareTo(options[j]) < 0 - }) - for _, o := range options { - args = append(args, o.Key+"="+o.Value) + args := append(options.Copy().Sort().AsArgs()) + if len(input.GroupSpec.Args) > 0 { + args = append(args, input.GroupSpec.Args...) } - args = append(args, svrSpec.Args...) return args } @@ -248,23 +191,20 @@ func createArangodArgs(apiObject metav1.Object, deplSpec api.DeploymentSpec, gro // createArangoSyncArgs creates command line arguments for an arangosync server in the given group. func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, groupSpec api.ServerGroupSpec, id string) []string { - options := make([]pod.OptionPair, 0, 64) + options := k8sutil.CreateOptionPairs(64) var runCmd string var port int /*if config.DebugCluster { options = append(options, - pod.OptionPair{"--log.level", "debug"}) + k8sutil.OptionPair{"--log.level", "debug"}) }*/ if spec.Sync.Monitoring.GetTokenSecretName() != "" { - options = append(options, - pod.OptionPair{"--monitoring.token", "$(" + constants.EnvArangoSyncMonitoringToken + ")"}, - ) + options.Addf("--monitoring.token", "$(%s)", constants.EnvArangoSyncMonitoringToken) } masterSecretPath := filepath.Join(k8sutil.MasterJWTSecretVolumeMountDir, constants.SecretKeyToken) - options = append(options, - pod.OptionPair{"--master.jwt-secret", masterSecretPath}, - ) + options.Add("--master.jwt-secret", masterSecretPath) + var masterEndpoint []string switch group { case api.ServerGroupSyncMasters: @@ -273,24 +213,19 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou masterEndpoint = spec.Sync.ExternalAccess.ResolveMasterEndpoint(k8sutil.CreateSyncMasterClientServiceDNSName(apiObject), port) keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) clientCAPath := filepath.Join(k8sutil.ClientAuthCAVolumeMountDir, constants.SecretCACertificate) - options = append(options, - pod.OptionPair{"--server.keyfile", keyPath}, - pod.OptionPair{"--server.client-cafile", clientCAPath}, - pod.OptionPair{"--mq.type", "direct"}, - ) + options.Add("--server.keyfile", keyPath) + options.Add("--server.client-cafile", clientCAPath) + options.Add("--mq.type", "direct") if spec.IsAuthenticated() { clusterSecretPath := filepath.Join(k8sutil.ClusterJWTSecretVolumeMountDir, constants.SecretKeyToken) - options = append(options, - pod.OptionPair{"--cluster.jwt-secret", clusterSecretPath}, - ) + options.Add("--cluster.jwt-secret", clusterSecretPath) } dbServiceName := k8sutil.CreateDatabaseClientServiceName(apiObject.GetName()) scheme := "http" if spec.IsSecure() { scheme = "https" } - options = append(options, - pod.OptionPair{"--cluster.endpoint", fmt.Sprintf("%s://%s:%d", scheme, dbServiceName, k8sutil.ArangoPort)}) + options.Addf("--cluster.endpoint", "%s://%s:%d", scheme, dbServiceName, k8sutil.ArangoPort) case api.ServerGroupSyncWorkers: runCmd = "worker" port = k8sutil.ArangoSyncWorkerPort @@ -298,24 +233,22 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou masterEndpoint = []string{"https://" + net.JoinHostPort(masterEndpointHost, strconv.Itoa(k8sutil.ArangoSyncMasterPort))} } for _, ep := range masterEndpoint { - options = append(options, - pod.OptionPair{"--master.endpoint", ep}) + options.Add("--master.endpoint", ep) } serverEndpoint := "https://" + net.JoinHostPort(k8sutil.CreatePodDNSName(apiObject, group.AsRole(), id), strconv.Itoa(port)) - options = append(options, - pod.OptionPair{"--server.endpoint", serverEndpoint}, - pod.OptionPair{"--server.port", strconv.Itoa(port)}, - ) - - args := make([]string, 0, 2+len(options)+len(groupSpec.Args)) - sort.Slice(options, func(i, j int) bool { - return options[i].CompareTo(options[j]) < 0 - }) - args = append(args, "run", runCmd) - for _, o := range options { - args = append(args, o.Key+"="+o.Value) + options.Add("--server.endpoint", serverEndpoint) + options.Add("--server.port", strconv.Itoa(port)) + + args := []string{ + "run", + runCmd, + } + + args = append(args, options.Sort().AsArgs()...) + + if len(groupSpec.Args) > 0 { + args = append(args, groupSpec.Args...) } - args = append(args, groupSpec.Args...) return args } @@ -392,7 +325,6 @@ func (r *Resources) RenderPodForMember(spec api.DeploymentSpec, status api.Deplo // Prepare arguments version := imageInfo.ArangoDBVersion autoUpgrade := m.Conditions.IsTrue(api.ConditionTypeAutoUpgrade) - args := createArangodArgs(apiObject, spec, group, status.Members.Agents, m.ID, version, autoUpgrade) tlsKeyfileSecretName := "" if spec.IsSecure() { @@ -402,18 +334,12 @@ func (r *Resources) RenderPodForMember(spec api.DeploymentSpec, status api.Deplo rocksdbEncryptionSecretName := "" if spec.RocksDB.IsEncrypted() { rocksdbEncryptionSecretName = spec.RocksDB.Encryption.GetKeySecretName() - if err := k8sutil.ValidateEncryptionKeySecret(secrets, rocksdbEncryptionSecretName); err != nil { - return nil, maskAny(errors.Wrapf(err, "RocksDB encryption key secret validation failed")) - } } - // Check cluster JWT secret + var clusterJWTSecretName string if spec.IsAuthenticated() { if versionHasJWTSecretKeyfile(version) { clusterJWTSecretName = spec.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, clusterJWTSecretName); err != nil { - return nil, maskAny(errors.Wrapf(err, "Cluster JWT secret validation failed")) - } } } @@ -428,6 +354,31 @@ func (r *Resources) RenderPodForMember(spec api.DeploymentSpec, status api.Deplo resources: r, imageInfo: imageInfo, context: r.context, + autoUpgrade: autoUpgrade, + deploymentStatus: status, + id: memberID, + } + + input := memberPod.AsInput() + + args := createArangodArgs(input) + + if err := memberPod.Validate(secrets); err != nil { + return nil, maskAny(errors.Wrapf(err, "Validation of pods resources failed")) + } + + if rocksdbEncryptionSecretName != "" { + if err := k8sutil.ValidateEncryptionKeySecret(secrets, rocksdbEncryptionSecretName); err != nil { + return nil, maskAny(errors.Wrapf(err, "RocksDB encryption key secret validation failed")) + } + } + + // Check cluster JWT secret + if clusterJWTSecretName != "" { + if err := k8sutil.ValidateTokenSecret(secrets, clusterJWTSecretName); err != nil { + return nil, maskAny(errors.Wrapf(err, "Cluster JWT secret validation failed")) + } + } return RenderArangoPod(apiObject, role, m.ID, m.PodName, args, &memberPod) diff --git a/pkg/deployment/resources/pod_creator_agent_args_test.go b/pkg/deployment/resources/pod_creator_agent_args_test.go index 015bb5be4..4825bf26b 100644 --- a/pkg/deployment/resources/pod_creator_agent_args_test.go +++ b/pkg/deployment/resources/pod_creator_agent_args_test.go @@ -25,6 +25,8 @@ package resources import ( "testing" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +53,18 @@ func TestCreateArangodArgsAgent(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupAgents, agents, "a1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupAgents, + GroupSpec: apiObject.Spec.Agents, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--agency.activate=true", @@ -94,7 +107,19 @@ func TestCreateArangodArgsAgent(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupAgents, agents, "a1", "", true) + + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupAgents, + GroupSpec: apiObject.Spec.Agents, + Version: "", + Enterprise: false, + AutoUpgrade: true, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--agency.activate=true", @@ -141,7 +166,19 @@ func TestCreateArangodArgsAgent(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupAgents, agents, "a1", "", false) + + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupAgents, + GroupSpec: apiObject.Spec.Agents, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--agency.activate=true", @@ -184,7 +221,18 @@ func TestCreateArangodArgsAgent(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupAgents, agents, "a1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupAgents, + GroupSpec: apiObject.Spec.Agents, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--agency.activate=true", @@ -227,7 +275,18 @@ func TestCreateArangodArgsAgent(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupAgents, agents, "a1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupAgents, + GroupSpec: apiObject.Spec.Agents, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--agency.activate=true", diff --git a/pkg/deployment/resources/pod_creator_arangod.go b/pkg/deployment/resources/pod_creator_arangod.go index 532246895..db0225376 100644 --- a/pkg/deployment/resources/pod_creator_arangod.go +++ b/pkg/deployment/resources/pod_creator_arangod.go @@ -52,10 +52,13 @@ type MemberArangoDPod struct { clusterJWTSecretName string groupSpec api.ServerGroupSpec spec api.DeploymentSpec + deploymentStatus api.DeploymentStatus group api.ServerGroup context Context resources *Resources imageInfo api.ImageInfo + autoUpgrade bool + id string } type ArangoDContainer struct { @@ -183,12 +186,35 @@ func (a *ArangoDContainer) GetImagePullPolicy() core.PullPolicy { return a.spec.GetImagePullPolicy() } +func (m *MemberArangoDPod) AsInput() pod.Input { + return pod.Input{ + ApiObject: m.context.GetAPIObject(), + Deployment: m.spec, + Status: m.deploymentStatus, + Group: m.group, + GroupSpec: m.groupSpec, + Version: m.imageInfo.ArangoDBVersion, + Enterprise: m.imageInfo.Enterprise, + AutoUpgrade: m.autoUpgrade, + ID: m.id, + } +} + func (m *MemberArangoDPod) Init(pod *core.Pod) { terminationGracePeriodSeconds := int64(math.Ceil(m.group.DefaultTerminationGracePeriod().Seconds())) pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds pod.Spec.PriorityClassName = m.groupSpec.PriorityClassName } +func (m *MemberArangoDPod) Validate(secrets k8sutil.SecretInterface) error { + i := m.AsInput() + if err := pod.SNI().Verify(i, secrets); err != nil { + return err + } + + return nil +} + func (m *MemberArangoDPod) GetName() string { return m.resources.context.GetAPIObject().GetName() } @@ -327,6 +353,19 @@ func (m *MemberArangoDPod) GetVolumes() ([]core.Volume, []core.VolumeMount) { volumes = append(volumes, k8sutil.LifecycleVolume()) } + // SNI + { + sniVolumes, sniVolumeMounts := pod.SNI().Volumes(m.AsInput()) + + if len(sniVolumes) > 0 { + volumes = append(volumes, sniVolumes...) + } + + if len(sniVolumeMounts) > 0 { + volumeMounts = append(volumeMounts, sniVolumeMounts...) + } + } + if len(m.groupSpec.Volumes) > 0 { volumes = append(volumes, m.groupSpec.Volumes.Volumes()...) } diff --git a/pkg/deployment/resources/pod_creator_coordinator_args_test.go b/pkg/deployment/resources/pod_creator_coordinator_args_test.go index b5cb2e10f..80112c9b2 100644 --- a/pkg/deployment/resources/pod_creator_coordinator_args_test.go +++ b/pkg/deployment/resources/pod_creator_coordinator_args_test.go @@ -25,6 +25,8 @@ package resources import ( "testing" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +53,18 @@ func TestCreateArangodArgsCoordinator(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupCoordinators, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupCoordinators, + GroupSpec: apiObject.Spec.Coordinators, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -92,7 +105,18 @@ func TestCreateArangodArgsCoordinator(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupCoordinators, agents, "id1", "", true) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupCoordinators, + GroupSpec: apiObject.Spec.Coordinators, + Version: "", + Enterprise: false, + AutoUpgrade: true, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -134,7 +158,18 @@ func TestCreateArangodArgsCoordinator(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupCoordinators, agents, "id1", "3.6.0", true) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupCoordinators, + GroupSpec: apiObject.Spec.Coordinators, + Version: "3.6.0", + Enterprise: false, + AutoUpgrade: true, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -179,7 +214,18 @@ func TestCreateArangodArgsCoordinator(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupCoordinators, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupCoordinators, + GroupSpec: apiObject.Spec.Coordinators, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=tcp://name-agent-a1.name-int.ns.svc:8529", @@ -219,7 +265,18 @@ func TestCreateArangodArgsCoordinator(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupCoordinators, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupCoordinators, + GroupSpec: apiObject.Spec.Coordinators, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -261,7 +318,18 @@ func TestCreateArangodArgsCoordinator(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupCoordinators, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupCoordinators, + GroupSpec: apiObject.Spec.Coordinators, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", diff --git a/pkg/deployment/resources/pod_creator_dbserver_args_test.go b/pkg/deployment/resources/pod_creator_dbserver_args_test.go index d536b6bb3..ca4e0532e 100644 --- a/pkg/deployment/resources/pod_creator_dbserver_args_test.go +++ b/pkg/deployment/resources/pod_creator_dbserver_args_test.go @@ -25,6 +25,8 @@ package resources import ( "testing" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -51,7 +53,18 @@ func TestCreateArangodArgsDBServer(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupDBServers, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupDBServers, + GroupSpec: apiObject.Spec.DBServers, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -92,7 +105,18 @@ func TestCreateArangodArgsDBServer(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupDBServers, agents, "id1", "", true) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupDBServers, + GroupSpec: apiObject.Spec.DBServers, + Version: "", + Enterprise: false, + AutoUpgrade: true, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -137,7 +161,18 @@ func TestCreateArangodArgsDBServer(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupDBServers, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupDBServers, + GroupSpec: apiObject.Spec.DBServers, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=tcp://name-agent-a1.name-int.ns.svc:8529", @@ -177,7 +212,18 @@ func TestCreateArangodArgsDBServer(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupDBServers, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupDBServers, + GroupSpec: apiObject.Spec.DBServers, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", @@ -219,7 +265,18 @@ func TestCreateArangodArgsDBServer(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupDBServers, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupDBServers, + GroupSpec: apiObject.Spec.DBServers, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", diff --git a/pkg/deployment/resources/pod_creator_single_args_test.go b/pkg/deployment/resources/pod_creator_single_args_test.go index 3dc128398..8e3eb3bf2 100644 --- a/pkg/deployment/resources/pod_creator_single_args_test.go +++ b/pkg/deployment/resources/pod_creator_single_args_test.go @@ -25,6 +25,8 @@ package resources import ( "testing" + "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -42,7 +44,17 @@ func TestCreateArangodArgsSingle(t *testing.T) { }, } apiObject.Spec.SetDefaults("test") - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, nil, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--database.directory=/data", @@ -69,7 +81,17 @@ func TestCreateArangodArgsSingle(t *testing.T) { }, } apiObject.Spec.SetDefaults("test") - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, nil, "id1", "", true) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: true, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--database.auto-upgrade=true", @@ -100,7 +122,17 @@ func TestCreateArangodArgsSingle(t *testing.T) { }, } apiObject.Spec.SetDefaults("test") - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, nil, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--database.directory=/data", @@ -126,7 +158,17 @@ func TestCreateArangodArgsSingle(t *testing.T) { }, } apiObject.Spec.SetDefaults("test") - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, nil, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--database.directory=/data", @@ -154,7 +196,17 @@ func TestCreateArangodArgsSingle(t *testing.T) { } apiObject.Spec.Authentication.JWTSecretName = util.NewString("None") apiObject.Spec.SetDefaults("test") - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, nil, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--database.directory=/data", @@ -181,7 +233,17 @@ func TestCreateArangodArgsSingle(t *testing.T) { } apiObject.Spec.Single.Args = []string{"--foo1", "--foo2"} apiObject.Spec.SetDefaults("test") - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, nil, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "a1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--database.directory=/data", @@ -219,7 +281,18 @@ func TestCreateArangodArgsSingle(t *testing.T) { api.MemberStatus{ID: "a2"}, api.MemberStatus{ID: "a3"}, } - cmdline := createArangodArgs(apiObject, apiObject.Spec, api.ServerGroupSingle, agents, "id1", "", false) + input := pod.Input{ + ApiObject: apiObject, + Deployment: apiObject.Spec, + Status: api.DeploymentStatus{Members: api.DeploymentStatusMembers{Agents: agents}}, + Group: api.ServerGroupSingle, + GroupSpec: apiObject.Spec.Single, + Version: "", + Enterprise: false, + AutoUpgrade: false, + ID: "id1", + } + cmdline := createArangodArgs(input) assert.Equal(t, []string{ "--cluster.agency-endpoint=ssl://name-agent-a1.name-int.ns.svc:8529", diff --git a/pkg/deployment/resources/pod_creator_sync.go b/pkg/deployment/resources/pod_creator_sync.go index 20a1405ff..096d60258 100644 --- a/pkg/deployment/resources/pod_creator_sync.go +++ b/pkg/deployment/resources/pod_creator_sync.go @@ -292,3 +292,7 @@ func (m *MemberSyncPod) Init(pod *core.Pod) { pod.Spec.TerminationGracePeriodSeconds = &terminationGracePeriodSeconds pod.Spec.PriorityClassName = m.groupSpec.PriorityClassName } + +func (m *MemberSyncPod) Validate(secrets k8sutil.SecretInterface) error { + return nil +} diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index 033fd748c..e7daacb52 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -29,7 +29,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - driver "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/metrics" "github.com/arangodb/kube-arangodb/pkg/util" @@ -276,16 +275,3 @@ func (r *Resources) InspectPods(ctx context.Context) (util.Interval, error) { } return nextInterval, nil } - -// GetExpectedPodArguments creates command line arguments for a server in the given group with given ID. -func (r *Resources) GetExpectedPodArguments(apiObject metav1.Object, deplSpec api.DeploymentSpec, group api.ServerGroup, - agents api.MemberStatusList, id string, version driver.Version) []string { - if group.IsArangod() { - return createArangodArgs(apiObject, deplSpec, group, agents, id, version, false) - } - if group.IsArangosync() { - groupSpec := deplSpec.GetServerGroupSpec(group) - return createArangoSyncArgs(apiObject, deplSpec, group, groupSpec, id) - } - return nil -} diff --git a/pkg/deployment/tls/client.go b/pkg/deployment/tls/client.go new file mode 100644 index 000000000..7e4a10ca2 --- /dev/null +++ b/pkg/deployment/tls/client.go @@ -0,0 +1,112 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package tls + +import ( + "context" + "net/http" + + "github.com/arangodb/go-driver" +) + +type KeyFile struct { + PrivateKeyChecksum string `json:"privateKeySHA256,omitempty"` + Checksum string `json:"SHA256,omitempty"` + Certificates []string `json:"certificates,omitempty"` +} + +type DetailsResult struct { + KeyFile KeyFile `json:"keyfile,omitempty"` + SNI map[string]KeyFile `json:"SNI,omitempty"` +} + +type Details struct { + Result DetailsResult `json:"result,omitempty"` +} + +func NewClient(c driver.Connection) Client { + return &client{ + c: c, + } +} + +type Client interface { + GetTLS(ctx context.Context) (Details, error) + RefreshTLS(ctx context.Context) (Details, error) +} + +type client struct { + c driver.Connection +} + +func (c *client) parseResponse(response driver.Response) (Details, error) { + if err := response.CheckStatus(http.StatusOK); err != nil { + return Details{}, err + } + + var d Details + + if err := response.ParseBody("", &d); err != nil { + return Details{}, err + } + + return d, nil +} + +func (c *client) GetTLS(ctx context.Context) (Details, error) { + r, err := c.c.NewRequest(http.MethodGet, "/_admin/server/tls") + if err != nil { + return Details{}, err + } + + response, err := c.c.Do(ctx, r) + if err != nil { + return Details{}, err + } + + d, err := c.parseResponse(response) + if err != nil { + return Details{}, err + } + + return d, nil +} + +func (c *client) RefreshTLS(ctx context.Context) (Details, error) { + r, err := c.c.NewRequest(http.MethodPost, "/_admin/server/tls") + if err != nil { + return Details{}, err + } + + response, err := c.c.Do(ctx, r) + if err != nil { + return Details{}, err + } + + d, err := c.parseResponse(response) + if err != nil { + return Details{}, err + } + + return d, nil +} diff --git a/pkg/deployment/pod/pair.go b/pkg/util/dict.go similarity index 61% rename from pkg/deployment/pod/pair.go rename to pkg/util/dict.go index 3f7de3e98..f6f8417e4 100644 --- a/pkg/deployment/pod/pair.go +++ b/pkg/util/dict.go @@ -20,27 +20,27 @@ // Author Adam Janikowski // -package pod +package util -import "strings" +import ( + "reflect" + "sort" +) -// OptionPair key value pair builder -type OptionPair struct { - Key string - Value string -} +func SortKeys(m interface{}) []string { + if m == nil { + return []string{} + } + + q := reflect.ValueOf(m).MapKeys() -// CompareTo returns -1 if o < other, 0 if o == other, 1 otherwise -func (o OptionPair) CompareTo(other OptionPair) int { - rc := strings.Compare(o.Key, other.Key) - if rc < 0 { - return -1 - } else if rc > 0 { - return 1 + r := make([]string, len(q)) + + for id, v := range q { + r[id] = v.String() } - return strings.Compare(o.Value, other.Value) -} -func NewOptionPair(pairs ...OptionPair) []OptionPair { - return pairs + sort.Strings(r) + + return r } diff --git a/pkg/util/k8sutil/pair.go b/pkg/util/k8sutil/pair.go new file mode 100644 index 000000000..51735d0a5 --- /dev/null +++ b/pkg/util/k8sutil/pair.go @@ -0,0 +1,125 @@ +// +// DISCLAIMER +// +// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package k8sutil + +import ( + "fmt" + "sort" + "strings" +) + +func CreateOptionPairs(lens ...int) OptionPairs { + l := 16 + + if len(lens) > 0 { + l = lens[0] + } + + return make(OptionPairs, 0, l) +} + +// OptionPairs list of pair builder +type OptionPairs []OptionPair + +func (o *OptionPairs) Append(pairs ...OptionPair) { + if o == nil { + *o = pairs + return + } + + *o = append(*o, pairs...) +} + +func (o *OptionPairs) Addf(key, format string, i ...interface{}) { + o.Add(key, fmt.Sprintf(format, i...)) +} + +func (o *OptionPairs) Add(key, value string) { + o.Append(OptionPair{ + Key: key, + Value: value, + }) +} + +func (o *OptionPairs) Merge(pairs ...OptionPairs) { + for _, pair := range pairs { + if len(pair) == 0 { + continue + } + + o.Append(pair...) + } +} + +func (o OptionPairs) Copy() OptionPairs { + r := make(OptionPairs, len(o)) + + for id, option := range o { + r[id] = option + } + + return r +} + +func (o OptionPairs) Sort() OptionPairs { + sort.Slice(o, func(i, j int) bool { + return o[i].CompareTo(o[j]) < 0 + }) + + return o +} + +func (o OptionPairs) AsArgs() []string { + s := make([]string, len(o)) + + for id, pair := range o { + s[id] = pair.String() + } + + return s +} + +// OptionPair key value pair builder +type OptionPair struct { + Key string + Value string +} + +func (o OptionPair) String() string { + return fmt.Sprintf("%s=%s", o.Key, o.Value) +} + +// CompareTo returns -1 if o < other, 0 if o == other, 1 otherwise +func (o OptionPair) CompareTo(other OptionPair) int { + rc := strings.Compare(o.Key, other.Key) + if rc < 0 { + return -1 + } else if rc > 0 { + return 1 + } + return strings.Compare(o.Value, other.Value) +} + +func NewOptionPair(pairs ...OptionPair) OptionPairs { + return pairs +} diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 258825e21..39875bd6f 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -52,6 +52,7 @@ const ( ArangodVolumeMountDir = "/data" RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption" TLSKeyfileVolumeMountDir = "/secrets/tls" + TLSSNIKeyfileVolumeMountDir = "/secrets/sni" ClientAuthCAVolumeMountDir = "/secrets/client-auth/ca" ClusterJWTSecretVolumeMountDir = "/secrets/cluster/jwt" ExporterJWTVolumeMountDir = "/secrets/exporter/jwt" @@ -75,6 +76,7 @@ type PodCreator interface { GetContainerCreator() ContainerCreator GetImagePullSecrets() []string IsDeploymentMode() bool + Validate(secrets SecretInterface) error } type ContainerCreator interface {