From 4d05a5b34803d7ea9984e1bee8323df6d843a3ad Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 10:00:55 +0100 Subject: [PATCH 01/12] First version of arangodbexporter support. --- .../deployment/v1alpha/deployment_spec.go | 1 + pkg/apis/deployment/v1alpha/metrics_spec.go | 34 ++++++++++++++ .../v1alpha/zz_generated.deepcopy.go | 22 +++++++++ pkg/deployment/images.go | 2 +- pkg/deployment/resources/pod_creator.go | 46 ++++++++++++++++++- pkg/util/k8sutil/constants.go | 1 + pkg/util/k8sutil/pods.go | 43 ++++++++++++++++- 7 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 pkg/apis/deployment/v1alpha/metrics_spec.go diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index d36c8b568..0f75f78b9 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -61,6 +61,7 @@ type DeploymentSpec struct { TLS TLSSpec `json:"tls"` Sync SyncSpec `json:"sync"` License LicenseSpec `json:"license"` + Metrics MetricsSpec `json:"metrics"` Single ServerGroupSpec `json:"single"` Agents ServerGroupSpec `json:"agents"` diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go new file mode 100644 index 000000000..c1705a850 --- /dev/null +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -0,0 +1,34 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// + +package v1alpha + +import "github.com/arangodb/kube-arangodb/pkg/util" + +// MetricsSpec contains spec for arangodb exporter +type MetricsSpec struct { + Enabled *bool `json:"enabled,omitempty"` +} + +// IsEnabled returns whether metrics are enabled or not +func (s *MetricsSpec) IsEnabled() bool { + return util.BoolOrDefault(s.Enabled, false) +} diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 666fb48cc..5036d291c 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -250,6 +250,7 @@ func (in *DeploymentSpec) DeepCopyInto(out *DeploymentSpec) { in.TLS.DeepCopyInto(&out.TLS) in.Sync.DeepCopyInto(&out.Sync) in.License.DeepCopyInto(&out.License) + in.Metrics.DeepCopyInto(&out.Metrics) in.Single.DeepCopyInto(&out.Single) in.Agents.DeepCopyInto(&out.Agents) in.DBServers.DeepCopyInto(&out.DBServers) @@ -525,6 +526,27 @@ func (in MemberStatusList) DeepCopy() MemberStatusList { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { + *out = *in + if in.Enabled != nil { + in, out := &in.Enabled, &out.Enabled + *out = new(bool) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsSpec. +func (in *MetricsSpec) DeepCopy() *MetricsSpec { + if in == nil { + return nil + } + out := new(MetricsSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MonitoringSpec) DeepCopyInto(out *MonitoringSpec) { *out = *in diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index 85d633cae..d8a23ffa0 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -198,7 +198,7 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima } } if err := k8sutil.CreateArangodPod(ib.KubeCli, true, ib.APIObject, role, id, podName, "", image, "", "", ib.Spec.GetImagePullPolicy(), "", false, terminationGracePeriod, args, env, nil, nil, nil, - tolerations, serviceAccountName, "", "", "", nil); err != nil { + tolerations, serviceAccountName, "", "", "", nil, nil); err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, maskAny(err) } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 3f3542629..196a47670 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -329,6 +329,23 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou return args } +func createExporterArgs() []string { + options := make([]optionPair, 0, 64) + options = append(options, + optionPair{"--arangodb.jwtsecret", "$(" + constants.EnvArangodJWTSecret + ")"}, + optionPair{"--arangodb.endpoint=http://localhost:", strconv.Itoa(k8sutil.ArangoPort)}, + ) + args := make([]string, 0, 2+len(options)) + sort.Slice(options, func(i, j int) bool { + return options[i].CompareTo(options[j]) < 0 + }) + for _, o := range options { + args = append(args, o.Key+"="+o.Value) + } + + return args +} + // createLivenessProbe creates configuration for a liveness probe of a server in the given group. func (r *Resources) createLivenessProbe(spec api.DeploymentSpec, group api.ServerGroup) (*k8sutil.HTTPProbeConfig, error) { switch group { @@ -476,6 +493,16 @@ func (r *Resources) createPodTolerations(group api.ServerGroup, groupSpec api.Se return tolerations } +func createExporterLivenessProbe() *k8sutil.HTTPProbeConfig { + probeCfg := &k8sutil.HTTPProbeConfig{ + LocalPath: "/", + InitialDelaySeconds: 2, + PeriodSeconds: 2, + } + + return probeCfg +} + // createPodForMember creates all Pods listed in member status func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once) error { kubecli := r.context.GetKubeCli() @@ -583,12 +610,29 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, } } + var exporter *k8sutil.ArangodbExporterContainerConf + + if spec.Metrics.IsEnabled() { + if group == api.ServerGroupDBServers || group == api.ServerGroupCoordinators { + env := make(map[string]k8sutil.EnvValue) + env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{ + SecretName: spec.Authentication.GetJWTSecretName(), + SecretKey: constants.SecretKeyToken, + } + exporter = &k8sutil.ArangodbExporterContainerConf{ + Args: createExporterArgs(), + Env: env, + LivenessProbe: createExporterLivenessProbe(), + } + } + } + engine := spec.GetStorageEngine().AsArangoArgument() requireUUID := group == api.ServerGroupDBServers && m.IsInitialized finalizers := r.createPodFinalizers(group) if err := k8sutil.CreateArangodPod(kubecli, spec.IsDevelopment(), apiObject, role, m.ID, m.PodName, m.PersistentVolumeClaimName, imageInfo.ImageID, lifecycleImage, alpineImage, spec.GetImagePullPolicy(), engine, requireUUID, terminationGracePeriod, args, env, finalizers, livenessProbe, readinessProbe, tolerations, serviceAccountName, tlsKeyfileSecretName, rocksdbEncryptionSecretName, - clusterJWTSecretName, groupSpec.GetNodeSelector()); err != nil { + clusterJWTSecretName, groupSpec.GetNodeSelector(), exporter); err != nil { return maskAny(err) } log.Debug().Str("pod-name", m.PodName).Msg("Created pod") diff --git a/pkg/util/k8sutil/constants.go b/pkg/util/k8sutil/constants.go index d4524d201..0ae48e1dd 100644 --- a/pkg/util/k8sutil/constants.go +++ b/pkg/util/k8sutil/constants.go @@ -27,6 +27,7 @@ const ( ArangoPort = 8529 ArangoSyncMasterPort = 8629 ArangoSyncWorkerPort = 8729 + ArangoExporterPort = 9101 // K8s constants ClusterIPNone = "None" diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 37c60d4ad..fbb04e06b 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -40,6 +40,7 @@ const ( InitDataContainerName = "init-data" InitLifecycleContainerName = "init-lifecycle" ServerContainerName = "server" + ExporterContainerName = "exporter" arangodVolumeName = "arangod-data" tlsKeyfileVolumeName = "tls-keyfile" lifecycleVolumeName = "lifecycle" @@ -329,6 +330,32 @@ func arangosyncContainer(image string, imagePullPolicy v1.PullPolicy, args []str return c } +func arangodbexporterContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { + // THIS IS WORK IN PROGRESS + // IN THE END THE ARANGO DOCKERIMAGE WILL CONTAIN THE EXPORTER + c := v1.Container{ + Command: append([]string{"/app/arangodb-exporter"}, args...), + Name: ExporterContainerName, + Image: "arangodb/arangodb-exporter:0.1.3", + ImagePullPolicy: v1.PullIfNotPresent, + Ports: []v1.ContainerPort{ + { + Name: "exporter", + ContainerPort: int32(ArangoExporterPort), + Protocol: v1.ProtocolTCP, + }, + }, + } + for k, v := range env { + c.Env = append(c.Env, v.CreateEnvVar(k)) + } + if livenessProbe != nil { + c.LivenessProbe = livenessProbe.Create() + } + + return c +} + // newLifecycle creates a lifecycle structure with preStop handler. func newLifecycle() (*v1.Lifecycle, []v1.EnvVar, []v1.Volume, error) { binaryPath, err := os.Executable() @@ -410,6 +437,13 @@ func newPod(deploymentName, ns, role, id, podName string, finalizers []string, t return p } +// ArangodbExporterContainerConf contains configuration of the exporter container +type ArangodbExporterContainerConf struct { + Args []string + Env map[string]EnvValue + LivenessProbe *HTTPProbeConfig +} + // CreateArangodPod creates a Pod that runs `arangod`. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. @@ -418,7 +452,8 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy engine string, requireUUID bool, terminationGracePeriod time.Duration, args []string, env map[string]EnvValue, finalizers []string, livenessProbe *HTTPProbeConfig, readinessProbe *HTTPProbeConfig, tolerations []v1.Toleration, serviceAccountName string, - tlsKeyfileSecretName, rocksdbEncryptionSecretName string, clusterJWTSecretName string, nodeSelector map[string]string) error { + tlsKeyfileSecretName, rocksdbEncryptionSecretName string, clusterJWTSecretName string, nodeSelector map[string]string, + exporter *ArangodbExporterContainerConf) error { // Prepare basic pod p := newPod(deployment.GetName(), deployment.GetNamespace(), role, id, podName, finalizers, tolerations, serviceAccountName, nodeSelector) terminationGracePeriodSeconds := int64(math.Ceil(terminationGracePeriod.Seconds())) @@ -453,6 +488,12 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy } p.Spec.Containers = append(p.Spec.Containers, c) + // Add arangodb exporter container + if exporter != nil { + c = arangodbexporterContainer(image, imagePullPolicy, exporter.Args, exporter.Env, exporter.LivenessProbe) + p.Spec.Containers = append(p.Spec.Containers, c) + } + // Add UUID init container if alpineImage != "" { p.Spec.InitContainers = append(p.Spec.InitContainers, arangodInitContainer("uuid", id, engine, alpineImage, requireUUID)) From b2a000850ebcc286725aaa3a4e2f724274c7119b Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 12:01:15 +0100 Subject: [PATCH 02/12] Add option for different exporter docker image. --- .../deployment/v1alpha/deployment_spec.go | 8 ++++ pkg/apis/deployment/v1alpha/metrics_spec.go | 42 ++++++++++++++++++- .../v1alpha/zz_generated.deepcopy.go | 5 +++ pkg/deployment/resources/pod_creator.go | 5 +++ pkg/util/k8sutil/pods.go | 6 +-- 5 files changed, 62 insertions(+), 4 deletions(-) diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index 0f75f78b9..b28a43fa5 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -180,6 +180,7 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) { s.Coordinators.SetDefaults(ServerGroupCoordinators, s.GetMode().HasCoordinators(), s.GetMode()) s.SyncMasters.SetDefaults(ServerGroupSyncMasters, s.Sync.IsEnabled(), s.GetMode()) s.SyncWorkers.SetDefaults(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode()) + s.Metrics.SetDefaults() s.Chaos.SetDefaults() } @@ -218,6 +219,7 @@ func (s *DeploymentSpec) SetDefaultsFrom(source DeploymentSpec) { s.Coordinators.SetDefaultsFrom(source.Coordinators) s.SyncMasters.SetDefaultsFrom(source.SyncMasters) s.SyncWorkers.SetDefaultsFrom(source.SyncWorkers) + s.Metrics.SetDefaultsFrom(source.Metrics) s.Chaos.SetDefaultsFrom(source.Chaos) } @@ -272,6 +274,9 @@ func (s *DeploymentSpec) Validate() error { if err := s.SyncWorkers.Validate(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode(), s.GetEnvironment()); err != nil { return maskAny(err) } + if err := s.Metrics.Validate(); err != nil { + return maskAny(errors.Wrap(err, "spec.metrics")) + } if err := s.Chaos.Validate(); err != nil { return maskAny(errors.Wrap(err, "spec.chaos")) } @@ -333,5 +338,8 @@ func (s DeploymentSpec) ResetImmutableFields(target *DeploymentSpec) []string { if l := s.SyncWorkers.ResetImmutableFields(ServerGroupSyncWorkers, "syncworkers", &target.SyncWorkers); l != nil { resetFields = append(resetFields, l...) } + if l := s.Metrics.ResetImmutableFields("metrics", &target.Metrics); l != nil { + resetFields = append(resetFields, l...) + } return resetFields } diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index c1705a850..a1f4e819e 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -25,10 +25,50 @@ import "github.com/arangodb/kube-arangodb/pkg/util" // MetricsSpec contains spec for arangodb exporter type MetricsSpec struct { - Enabled *bool `json:"enabled,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Image *string `json:"image,omitempty"` + //Authentication struct { + // // JWTSecretName contains the name of the JWT kubernetes secret used for authentication + // JWTSecretName *string `json:"JWTSecretName,omitempty"` + //} `json:"authentication,omitempty"` } // IsEnabled returns whether metrics are enabled or not func (s *MetricsSpec) IsEnabled() bool { return util.BoolOrDefault(s.Enabled, false) } + +// HasImage returns whether a image was specified or not +func (s *MetricsSpec) HasImage() bool { + return s.Image != nil +} + +// GetImage returns the Image or empty string +func (s *MetricsSpec) GetImage() string { + return util.StringOrDefault(s.Image) +} + +// SetDefaults sets default values +func (s *MetricsSpec) SetDefaults() { + s.Enabled = util.NewBool(false) +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (s *MetricsSpec) SetDefaultsFrom(source MetricsSpec) { + if s.Enabled == nil { + s.Enabled = util.NewBoolOrNil(source.Enabled) + } + if s.Image == nil { + s.Image = util.NewStringOrNil(source.Image) + } +} + +// Validate the given spec +func (s *MetricsSpec) Validate() error { + return nil +} + +// ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. +func (s SyncSpec) ResetImmutableFields(fieldPrefix string, target *SyncSpec) []string { + return nil +} diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 5036d291c..2b7109903 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -534,6 +534,11 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { *out = new(bool) **out = **in } + if in.Image != nil { + in, out := &in.Image, &out.Image + *out = new(string) + **out = **in + } return } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 196a47670..30234a3d5 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -619,10 +619,15 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, SecretName: spec.Authentication.GetJWTSecretName(), SecretKey: constants.SecretKeyToken, } + image := spec.GetImage() + if spec.Metrics.HasImage() { + image = spec.Metrics.GetImage() + } exporter = &k8sutil.ArangodbExporterContainerConf{ Args: createExporterArgs(), Env: env, LivenessProbe: createExporterLivenessProbe(), + Image: image, } } } diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index fbb04e06b..d417953da 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -336,7 +336,7 @@ func arangodbexporterContainer(image string, imagePullPolicy v1.PullPolicy, args c := v1.Container{ Command: append([]string{"/app/arangodb-exporter"}, args...), Name: ExporterContainerName, - Image: "arangodb/arangodb-exporter:0.1.3", + Image: image, ImagePullPolicy: v1.PullIfNotPresent, Ports: []v1.ContainerPort{ { @@ -352,7 +352,6 @@ func arangodbexporterContainer(image string, imagePullPolicy v1.PullPolicy, args if livenessProbe != nil { c.LivenessProbe = livenessProbe.Create() } - return c } @@ -442,6 +441,7 @@ type ArangodbExporterContainerConf struct { Args []string Env map[string]EnvValue LivenessProbe *HTTPProbeConfig + Image string } // CreateArangodPod creates a Pod that runs `arangod`. @@ -490,7 +490,7 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy // Add arangodb exporter container if exporter != nil { - c = arangodbexporterContainer(image, imagePullPolicy, exporter.Args, exporter.Env, exporter.LivenessProbe) + c = arangodbexporterContainer(exporter.Image, imagePullPolicy, exporter.Args, exporter.Env, exporter.LivenessProbe) p.Spec.Containers = append(p.Spec.Containers, c) } From e1a0b7cbf45753440980a6e8d4164468b76d9a89 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 12:37:08 +0100 Subject: [PATCH 03/12] Rotate pods when exporter config changed. --- pkg/apis/deployment/v1alpha/metrics_spec.go | 6 ++++-- pkg/apis/deployment/v1alpha/server_group.go | 10 ++++++++++ pkg/deployment/reconcile/plan_builder.go | 18 ++++++++++++++++++ pkg/deployment/resources/pod_creator.go | 2 +- 4 files changed, 33 insertions(+), 3 deletions(-) diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index a1f4e819e..da7fc18a5 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -50,7 +50,9 @@ func (s *MetricsSpec) GetImage() string { // SetDefaults sets default values func (s *MetricsSpec) SetDefaults() { - s.Enabled = util.NewBool(false) + if s.Enabled == nil { + s.Enabled = util.NewBool(false) + } } // SetDefaultsFrom fills unspecified fields with a value from given source spec. @@ -69,6 +71,6 @@ func (s *MetricsSpec) Validate() error { } // ResetImmutableFields replaces all immutable fields in the given target with values from the source spec. -func (s SyncSpec) ResetImmutableFields(fieldPrefix string, target *SyncSpec) []string { +func (s MetricsSpec) ResetImmutableFields(fieldPrefix string, target *MetricsSpec) []string { return nil } diff --git a/pkg/apis/deployment/v1alpha/server_group.go b/pkg/apis/deployment/v1alpha/server_group.go index 4ccb7f21b..8d75043fe 100644 --- a/pkg/apis/deployment/v1alpha/server_group.go +++ b/pkg/apis/deployment/v1alpha/server_group.go @@ -130,3 +130,13 @@ func (g ServerGroup) IsArangosync() bool { return false } } + +// IsExportMetrics return true when the group can be used with the arangodbexporter +func (g ServerGroup) IsExportMetrics() bool { + switch g { + case ServerGroupCoordinators, ServerGroupDBServers: + return true + default: + return false + } +} diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 8a73ea388..80068940a 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -324,6 +324,24 @@ func podNeedsRotation(log zerolog.Logger, p v1.Pod, apiObject metav1.Object, spe return false, "Server Image not found" } + if group.IsExportMetrics() { + e, hasExporter := k8sutil.GetContainerByName(&p, k8sutil.ExporterContainerName) + + if spec.Metrics.IsEnabled() { + if !hasExporter { + return true, "Exporter configuration changed" + } + + if spec.Metrics.HasImage() { + if e.Image != spec.Metrics.GetImage() { + return true, "Exporter image changed" + } + } + } else if hasExporter { + return true, "Exporter was disabled" + } + } + // Check arguments expectedArgs := strings.Join(context.GetExpectedPodArguments(apiObject, spec, group, status.Members.Agents, id, podImageInfo.ArangoDBVersion), " ") actualArgs := strings.Join(getContainerArgs(c), " ") diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 30234a3d5..aec53ded6 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -613,7 +613,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, var exporter *k8sutil.ArangodbExporterContainerConf if spec.Metrics.IsEnabled() { - if group == api.ServerGroupDBServers || group == api.ServerGroupCoordinators { + if group.IsExportMetrics() { env := make(map[string]k8sutil.EnvValue) env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{ SecretName: spec.Authentication.GetJWTSecretName(), From fbbce0fd1be2a7699e6cf7a315bcecdeeab5292a Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 15:12:26 +0100 Subject: [PATCH 04/12] Added JWT for exporter as file. --- .../deployment/v1alpha/deployment_spec.go | 2 +- pkg/apis/deployment/v1alpha/metrics_spec.go | 45 +++++++++++++++---- .../v1alpha/zz_generated.deepcopy.go | 22 +++++++++ pkg/deployment/resources/pod_creator.go | 18 +++----- pkg/deployment/resources/secrets.go | 31 +++++++++++++ pkg/util/k8sutil/pods.go | 39 +++++++++++++--- pkg/util/k8sutil/secrets.go | 22 +++++++++ 7 files changed, 153 insertions(+), 26 deletions(-) diff --git a/pkg/apis/deployment/v1alpha/deployment_spec.go b/pkg/apis/deployment/v1alpha/deployment_spec.go index b28a43fa5..e6cdf4a18 100644 --- a/pkg/apis/deployment/v1alpha/deployment_spec.go +++ b/pkg/apis/deployment/v1alpha/deployment_spec.go @@ -180,7 +180,7 @@ func (s *DeploymentSpec) SetDefaults(deploymentName string) { s.Coordinators.SetDefaults(ServerGroupCoordinators, s.GetMode().HasCoordinators(), s.GetMode()) s.SyncMasters.SetDefaults(ServerGroupSyncMasters, s.Sync.IsEnabled(), s.GetMode()) s.SyncWorkers.SetDefaults(ServerGroupSyncWorkers, s.Sync.IsEnabled(), s.GetMode()) - s.Metrics.SetDefaults() + s.Metrics.SetDefaults(deploymentName+"-exporter-jwt-token", s.Authentication.IsAuthenticated()) s.Chaos.SetDefaults() } diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index da7fc18a5..41c316bb5 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -21,16 +21,22 @@ package v1alpha -import "github.com/arangodb/kube-arangodb/pkg/util" +import ( + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" +) + +// MetricsAuthenticationSpec contains spec for authentication with arangodb +type MetricsAuthenticationSpec struct { + // JWTTokenSecretName contains the name of the JWT kubernetes secret used for authentication + JWTTokenSecretName *string `json:"jwtTokenSecretName,omitempty"` +} // MetricsSpec contains spec for arangodb exporter type MetricsSpec struct { - Enabled *bool `json:"enabled,omitempty"` - Image *string `json:"image,omitempty"` - //Authentication struct { - // // JWTSecretName contains the name of the JWT kubernetes secret used for authentication - // JWTSecretName *string `json:"JWTSecretName,omitempty"` - //} `json:"authentication,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + Image *string `json:"image,omitempty"` + Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"` } // IsEnabled returns whether metrics are enabled or not @@ -49,10 +55,23 @@ func (s *MetricsSpec) GetImage() string { } // SetDefaults sets default values -func (s *MetricsSpec) SetDefaults() { +func (s *MetricsSpec) SetDefaults(defaultTokenName string, isAuthenticated bool) { if s.Enabled == nil { s.Enabled = util.NewBool(false) } + if s.GetJWTTokenSecretName() == "" { + s.Authentication.JWTTokenSecretName = util.NewString(defaultTokenName) + } +} + +// GetJWTTokenSecretName returns the token secret name or empty string +func (s *MetricsSpec) GetJWTTokenSecretName() string { + return util.StringOrDefault(s.Authentication.JWTTokenSecretName) +} + +// HasJWTTokenSecretName returns true if a secret name was specified +func (s *MetricsSpec) HasJWTTokenSecretName() bool { + return s.Authentication.JWTTokenSecretName != nil } // SetDefaultsFrom fills unspecified fields with a value from given source spec. @@ -63,10 +82,20 @@ func (s *MetricsSpec) SetDefaultsFrom(source MetricsSpec) { if s.Image == nil { s.Image = util.NewStringOrNil(source.Image) } + if s.Authentication.JWTTokenSecretName == nil { + s.Authentication.JWTTokenSecretName = util.NewStringOrNil(source.Authentication.JWTTokenSecretName) + } } // Validate the given spec func (s *MetricsSpec) Validate() error { + + if s.HasJWTTokenSecretName() { + if err := k8sutil.ValidateResourceName(s.GetJWTTokenSecretName()); err != nil { + return err + } + } + return nil } diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 2b7109903..8dcf0dd78 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -526,6 +526,27 @@ func (in MemberStatusList) DeepCopy() MemberStatusList { return *out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsAuthenticationSpec) DeepCopyInto(out *MetricsAuthenticationSpec) { + *out = *in + if in.JWTTokenSecretName != nil { + in, out := &in.JWTTokenSecretName, &out.JWTTokenSecretName + *out = new(string) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsAuthenticationSpec. +func (in *MetricsAuthenticationSpec) DeepCopy() *MetricsAuthenticationSpec { + if in == nil { + return nil + } + out := new(MetricsAuthenticationSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { *out = *in @@ -539,6 +560,7 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { *out = new(string) **out = **in } + in.Authentication.DeepCopyInto(&out.Authentication) return } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index aec53ded6..00b83cae5 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -330,10 +330,11 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou } func createExporterArgs() []string { + tokenpath := filepath.Join(k8sutil.ExporterJWTVolumeMountDir, constants.SecretKeyToken) options := make([]optionPair, 0, 64) options = append(options, - optionPair{"--arangodb.jwtsecret", "$(" + constants.EnvArangodJWTSecret + ")"}, - optionPair{"--arangodb.endpoint=http://localhost:", strconv.Itoa(k8sutil.ArangoPort)}, + optionPair{"--arangodb.jwt-file", tokenpath}, + optionPair{"--arangodb.endpoint", "http://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, ) args := make([]string, 0, 2+len(options)) sort.Slice(options, func(i, j int) bool { @@ -614,20 +615,15 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, if spec.Metrics.IsEnabled() { if group.IsExportMetrics() { - env := make(map[string]k8sutil.EnvValue) - env[constants.EnvArangodJWTSecret] = k8sutil.EnvValue{ - SecretName: spec.Authentication.GetJWTSecretName(), - SecretKey: constants.SecretKeyToken, - } image := spec.GetImage() if spec.Metrics.HasImage() { image = spec.Metrics.GetImage() } exporter = &k8sutil.ArangodbExporterContainerConf{ - Args: createExporterArgs(), - Env: env, - LivenessProbe: createExporterLivenessProbe(), - Image: image, + Args: createExporterArgs(), + JWTTokenSecretName: spec.Metrics.GetJWTTokenSecretName(), + LivenessProbe: createExporterLivenessProbe(), + Image: image, } } } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 001669130..e902b8258 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -55,6 +55,12 @@ func (r *Resources) EnsureSecrets() error { if err := r.ensureTokenSecret(secrets, spec.Authentication.GetJWTSecretName()); err != nil { return maskAny(err) } + + if spec.Metrics.IsEnabled() { + if err := r.ensureExporterTokenSecret(secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName()); err != nil { + return maskAny(err) + } + } } if spec.IsSecure() { counterMetric.Inc() @@ -110,6 +116,31 @@ func (r *Resources) ensureTokenSecret(secrets k8sutil.SecretInterface, secretNam return nil } +// ensureExporterTokenSecret checks if a secret with given name exists in the namespace +// of the deployment. If not, it will add such a secret with correct access. +func (r *Resources) ensureExporterTokenSecret(secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) error { + if _, err := secrets.Get(tokenSecretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) { + // Secret not found, create it + claims := map[string]interface{}{ + "iss": "arangodb", + "server_id": "exporter", + } + // Create secret + owner := r.context.GetAPIObject().AsOwner() + if err := k8sutil.CreateJWTFromSecret(secrets, tokenSecretName, secretSecretName, claims, &owner); k8sutil.IsAlreadyExists(err) { + // Secret added while we tried it also + return nil + } else if err != nil { + // Failed to create secret + return maskAny(err) + } + } else if err != nil { + // Failed to get secret for other reasons + return maskAny(err) + } + return nil +} + // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. func (r *Resources) ensureTLSCACertificateSecret(secrets k8sutil.SecretInterface, spec api.TLSSpec) error { diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index d417953da..7e3794bf3 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -48,6 +48,7 @@ const ( clusterJWTSecretVolumeName = "cluster-jwt" masterJWTSecretVolumeName = "master-jwt" rocksdbEncryptionVolumeName = "rocksdb-encryption" + exporterJWTVolumeName = "exporter-jwt" ArangodVolumeMountDir = "/data" RocksDBEncryptionVolumeMountDir = "/secrets/rocksdb/encryption" JWTSecretFileVolumeMountDir = "/secrets/jwt" @@ -55,6 +56,7 @@ const ( LifecycleVolumeMountDir = "/lifecycle/tools" ClientAuthCAVolumeMountDir = "/secrets/client-auth/ca" ClusterJWTSecretVolumeMountDir = "/secrets/cluster/jwt" + ExporterJWTVolumeMountDir = "/secrets/exporter/jwt" MasterJWTSecretVolumeMountDir = "/secrets/master/jwt" ) @@ -222,6 +224,15 @@ func clusterJWTVolumeMounts() []v1.VolumeMount { } } +func exporterJWTVolumeMounts() []v1.VolumeMount { + return []v1.VolumeMount{ + { + Name: exporterJWTVolumeName, + MountPath: ExporterJWTVolumeMountDir, + }, + } +} + // rocksdbEncryptionVolumeMounts creates a volume mount structure for a RocksDB encryption key. func rocksdbEncryptionVolumeMounts() []v1.VolumeMount { return []v1.VolumeMount{ @@ -331,8 +342,6 @@ func arangosyncContainer(image string, imagePullPolicy v1.PullPolicy, args []str } func arangodbexporterContainer(image string, imagePullPolicy v1.PullPolicy, args []string, env map[string]EnvValue, livenessProbe *HTTPProbeConfig) v1.Container { - // THIS IS WORK IN PROGRESS - // IN THE END THE ARANGO DOCKERIMAGE WILL CONTAIN THE EXPORTER c := v1.Container{ Command: append([]string{"/app/arangodb-exporter"}, args...), Name: ExporterContainerName, @@ -438,10 +447,11 @@ func newPod(deploymentName, ns, role, id, podName string, finalizers []string, t // ArangodbExporterContainerConf contains configuration of the exporter container type ArangodbExporterContainerConf struct { - Args []string - Env map[string]EnvValue - LivenessProbe *HTTPProbeConfig - Image string + Args []string + Env map[string]EnvValue + JWTTokenSecretName string + LivenessProbe *HTTPProbeConfig + Image string } // CreateArangodPod creates a Pod that runs `arangod`. @@ -486,11 +496,15 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy if clusterJWTSecretName != "" { c.VolumeMounts = append(c.VolumeMounts, clusterJWTVolumeMounts()...) } + p.Spec.Containers = append(p.Spec.Containers, c) // Add arangodb exporter container if exporter != nil { c = arangodbexporterContainer(exporter.Image, imagePullPolicy, exporter.Args, exporter.Env, exporter.LivenessProbe) + if exporter.JWTTokenSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, exporterJWTVolumeMounts()...) + } p.Spec.Containers = append(p.Spec.Containers, c) } @@ -548,6 +562,19 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy p.Spec.Volumes = append(p.Spec.Volumes, vol) } + // Exporter Token Mount + if exporter != nil && exporter.JWTTokenSecretName != "" { + vol := v1.Volume{ + Name: exporterJWTVolumeName, + VolumeSource: v1.VolumeSource{ + Secret: &v1.SecretVolumeSource{ + SecretName: exporter.JWTTokenSecretName, + }, + }, + } + p.Spec.Volumes = append(p.Spec.Volumes, vol) + } + // Cluster JWT secret mount (if any) if clusterJWTSecretName != "" { vol := v1.Volume{ diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 6c32f837a..602706ec1 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -29,6 +29,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/arangodb/kube-arangodb/pkg/util/constants" + jg "github.com/dgrijalva/jwt-go" ) // SecretInterface has methods to work with Secret resources. @@ -248,6 +249,27 @@ func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerR return nil } +// CreateJWTFromSecret creates a JWT using the secret stored in secretSecretName and stores the +// result in a new secret called tokenSecretName +func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *metav1.OwnerReference) error { + + secret, err := GetTokenSecret(secrets, secretSecretName) + if err != nil { + return maskAny(err) + } + // Create a new token object, specifying signing method and the claims + // you would like it to contain. + token := jg.NewWithClaims(jg.SigningMethodHS256, jg.MapClaims(claims)) + + // Sign and get the complete encoded token as a string using the secret + signedToken, err := token.SignedString([]byte(secret)) + if err != nil { + return maskAny(err) + } + + return CreateTokenSecret(secrets, tokenSecretName, signedToken, ownerRef) +} + // GetBasicAuthSecret loads a secret with given name in the given namespace // and extracts the `username` & `password` field. // If the secret does not exists or one of the fields is missing, From fb15f899fe2a0e21b8c3266fac3a0a755745abea Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 15:29:38 +0100 Subject: [PATCH 05/12] Extended lifeness checks for exporter. --- pkg/deployment/resources/pod_creator.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 00b83cae5..5ed98be25 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -497,8 +497,9 @@ func (r *Resources) createPodTolerations(group api.ServerGroup, groupSpec api.Se func createExporterLivenessProbe() *k8sutil.HTTPProbeConfig { probeCfg := &k8sutil.HTTPProbeConfig{ LocalPath: "/", - InitialDelaySeconds: 2, - PeriodSeconds: 2, + InitialDelaySeconds: 5, + PeriodSeconds: 30, + TimeoutSeconds: 5, } return probeCfg From cec4ceb8dbc6c879411a49c29e475bda9a44b327 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 15:57:11 +0100 Subject: [PATCH 06/12] Use correct port for liveness of exporter. --- pkg/deployment/resources/pod_creator.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 5ed98be25..7af76b5b9 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -496,10 +496,8 @@ func (r *Resources) createPodTolerations(group api.ServerGroup, groupSpec api.Se func createExporterLivenessProbe() *k8sutil.HTTPProbeConfig { probeCfg := &k8sutil.HTTPProbeConfig{ - LocalPath: "/", - InitialDelaySeconds: 5, - PeriodSeconds: 30, - TimeoutSeconds: 5, + LocalPath: "/", + Port: k8sutil.ArangoExporterPort, } return probeCfg From abad00bbecbd9cac7300aee5fa9bd31780c39504 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Thu, 10 Jan 2019 17:14:49 +0100 Subject: [PATCH 07/12] Updated success and fail condition to only look at arangod. --- pkg/util/k8sutil/pods.go | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 7e3794bf3..e5af3193f 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -97,13 +97,42 @@ func IsPodReady(pod *v1.Pod) bool { // IsPodSucceeded returns true if all containers of the pod // have terminated with exit code 0. func IsPodSucceeded(pod *v1.Pod) bool { - return pod.Status.Phase == v1.PodSucceeded + if pod.Status.Phase == v1.PodSucceeded { + return true + } else { + for _, c := range pod.Status.ContainerStatuses { + if c.Name != "server" { + continue + } + + t := c.State.Terminated + if t != nil { + return t.ExitCode == 0 + } + } + return false + } } // IsPodFailed returns true if all containers of the pod // have terminated and at least one of them wih a non-zero exit code. func IsPodFailed(pod *v1.Pod) bool { - return pod.Status.Phase == v1.PodFailed + if pod.Status.Phase == v1.PodFailed { + return true + } else { + for _, c := range pod.Status.ContainerStatuses { + if c.Name != "server" { + continue + } + + t := c.State.Terminated + if t != nil { + return t.ExitCode != 0 + } + } + + return false + } } // IsPodScheduled returns true if the pod has been scheduled. From e4eff49871a409fb9b4f8210ce7050282c168f54 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Fri, 11 Jan 2019 10:10:40 +0100 Subject: [PATCH 08/12] Added TLS support for exporter using aragod certs. --- pkg/deployment/resources/pod_creator.go | 15 +++++++++++---- pkg/util/k8sutil/pods.go | 7 +++++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index 7af76b5b9..bfa709925 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -329,13 +329,19 @@ func createArangoSyncArgs(apiObject metav1.Object, spec api.DeploymentSpec, grou return args } -func createExporterArgs() []string { +func createExporterArgs(isSecure bool) []string { tokenpath := filepath.Join(k8sutil.ExporterJWTVolumeMountDir, constants.SecretKeyToken) options := make([]optionPair, 0, 64) options = append(options, optionPair{"--arangodb.jwt-file", tokenpath}, optionPair{"--arangodb.endpoint", "http://localhost:" + strconv.Itoa(k8sutil.ArangoPort)}, ) + keyPath := filepath.Join(k8sutil.TLSKeyfileVolumeMountDir, constants.SecretTLSKeyfile) + if isSecure { + options = append(options, + optionPair{"--ssl.keyfile", keyPath}, + ) + } args := make([]string, 0, 2+len(options)) sort.Slice(options, func(i, j int) bool { return options[i].CompareTo(options[j]) < 0 @@ -494,10 +500,11 @@ func (r *Resources) createPodTolerations(group api.ServerGroup, groupSpec api.Se return tolerations } -func createExporterLivenessProbe() *k8sutil.HTTPProbeConfig { +func createExporterLivenessProbe(isSecure bool) *k8sutil.HTTPProbeConfig { probeCfg := &k8sutil.HTTPProbeConfig{ LocalPath: "/", Port: k8sutil.ArangoExporterPort, + Secure: isSecure, } return probeCfg @@ -619,9 +626,9 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, image = spec.Metrics.GetImage() } exporter = &k8sutil.ArangodbExporterContainerConf{ - Args: createExporterArgs(), + Args: createExporterArgs(spec.IsSecure()), JWTTokenSecretName: spec.Metrics.GetJWTTokenSecretName(), - LivenessProbe: createExporterLivenessProbe(), + LivenessProbe: createExporterLivenessProbe(spec.IsSecure()), Image: image, } } diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index e5af3193f..e838d686a 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -101,7 +101,7 @@ func IsPodSucceeded(pod *v1.Pod) bool { return true } else { for _, c := range pod.Status.ContainerStatuses { - if c.Name != "server" { + if c.Name != ServerContainerName { continue } @@ -121,7 +121,7 @@ func IsPodFailed(pod *v1.Pod) bool { return true } else { for _, c := range pod.Status.ContainerStatuses { - if c.Name != "server" { + if c.Name != ServerContainerName { continue } @@ -534,6 +534,9 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy if exporter.JWTTokenSecretName != "" { c.VolumeMounts = append(c.VolumeMounts, exporterJWTVolumeMounts()...) } + if tlsKeyfileSecretName != "" { + c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) + } p.Spec.Containers = append(p.Spec.Containers, c) } From c74d83819d09ed3c5d617b2793e99490f9764680 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Fri, 11 Jan 2019 13:42:35 +0100 Subject: [PATCH 09/12] API layout for creating prometheus ServiceMonitor. --- pkg/apis/deployment/v1alpha/metrics_spec.go | 96 +++++++++++++++++++ pkg/apis/deployment/v1alpha/server_group.go | 2 +- .../v1alpha/zz_generated.deepcopy.go | 22 +++++ pkg/deployment/resources/monitoring.go | 50 ++++++++++ 4 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 pkg/deployment/resources/monitoring.go diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index 41c316bb5..84845b976 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -24,6 +24,7 @@ package v1alpha import ( "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/pkg/errors" ) // MetricsAuthenticationSpec contains spec for authentication with arangodb @@ -32,11 +33,44 @@ type MetricsAuthenticationSpec struct { JWTTokenSecretName *string `json:"jwtTokenSecretName,omitempty"` } +// MetricsMonitorResourceMode specifies the type of ServiceMonitor resource to be installed +type MetricsMonitorResourceMode string + +const ( + // MetricsMonitorResourceModeAuto detects if the CRD for ServiceMonitor is available and creates the resource if so (default) + MetricsMonitorResourceModeAuto MetricsMonitorResourceMode = "Auto" + // MetricsMonitorResourceModeNone never creates the resource + MetricsMonitorResourceModeNone MetricsMonitorResourceMode = "None" + // MetricsMonitorResourceModeAlways always creates the resource and fails if the CRD is not available + MetricsMonitorResourceModeAlways MetricsMonitorResourceMode = "Always" +) + +// IsAuto returns true if mode is MetricsMonitorResourceModeAuto +func (t MetricsMonitorResourceMode) IsAuto() bool { + return t == MetricsMonitorResourceModeAuto +} + +// IsNone returns true if mode is MetricsMonitorResourceModeNone +func (t MetricsMonitorResourceMode) IsNone() bool { + return t == MetricsMonitorResourceModeNone +} + +// IsAlways returns true if mode is MetricsMonitorResourceModeAlways +func (t MetricsMonitorResourceMode) IsAlways() bool { + return t == MetricsMonitorResourceModeAlways +} + +// MetricsPrometheusSpec contains metrics for prometheus +type MetricsPrometheusSpec struct { + MonitorResource *MetricsMonitorResourceMode `json:"monitorResource,omitempty"` +} + // MetricsSpec contains spec for arangodb exporter type MetricsSpec struct { Enabled *bool `json:"enabled,omitempty"` Image *string `json:"image,omitempty"` Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"` + Prometheus MetricsPrometheusSpec `json:"prometheus,omitempty"` } // IsEnabled returns whether metrics are enabled or not @@ -62,6 +96,7 @@ func (s *MetricsSpec) SetDefaults(defaultTokenName string, isAuthenticated bool) if s.GetJWTTokenSecretName() == "" { s.Authentication.JWTTokenSecretName = util.NewString(defaultTokenName) } + s.Prometheus.SetDefaults() } // GetJWTTokenSecretName returns the token secret name or empty string @@ -85,6 +120,7 @@ func (s *MetricsSpec) SetDefaultsFrom(source MetricsSpec) { if s.Authentication.JWTTokenSecretName == nil { s.Authentication.JWTTokenSecretName = util.NewStringOrNil(source.Authentication.JWTTokenSecretName) } + s.Prometheus.SetDefaultsFrom(source.Prometheus) } // Validate the given spec @@ -96,6 +132,10 @@ func (s *MetricsSpec) Validate() error { } } + if err := s.Prometheus.Validate(); err != nil { + return err + } + return nil } @@ -103,3 +143,59 @@ func (s *MetricsSpec) Validate() error { func (s MetricsSpec) ResetImmutableFields(fieldPrefix string, target *MetricsSpec) []string { return nil } + +// SetDefaults sets default values +func (p *MetricsPrometheusSpec) SetDefaults() { +} + +// SetDefaultsFrom fills unspecified fields with a value from given source spec. +func (p *MetricsPrometheusSpec) SetDefaultsFrom(source MetricsPrometheusSpec) { + if p.MonitorResource == nil { + p.MonitorResource = NewMetricsMonitorResourceModeOrNil(source.MonitorResource) + } +} + +// Validate the given spec +func (p *MetricsPrometheusSpec) Validate() error { + return p.MonitorResource.Validate() +} + +// Validate the given spec +func (t MetricsMonitorResourceMode) Validate() error { + switch t { + case MetricsMonitorResourceModeAuto, MetricsMonitorResourceModeNone, MetricsMonitorResourceModeAlways: + return nil + default: + return maskAny(errors.Wrapf(ValidationError, "Unknown monitor resource mode: '%s'", string(t))) + } +} + +// NewMetricsMonitorResourceMode returns a reference to a string with given value. +func NewMetricsMonitorResourceMode(input MetricsMonitorResourceMode) *MetricsMonitorResourceMode { + return &input +} + +// NewMetricsMonitorResourceModeOrNil returns nil if input is nil, otherwise returns a clone of the given value. +func NewMetricsMonitorResourceModeOrNil(input *MetricsMonitorResourceMode) *MetricsMonitorResourceMode { + if input == nil { + return nil + } + + return NewMetricsMonitorResourceMode(*input) +} + +// MetricsMonitorResourceModeOrDefault returns the default value (or empty string) if input is nil, otherwise returns the referenced value. +func MetricsMonitorResourceModeOrDefault(input *MetricsMonitorResourceMode, defaultValue ...MetricsMonitorResourceMode) MetricsMonitorResourceMode { + if input == nil { + if len(defaultValue) > 0 { + return defaultValue[0] + } + return "" + } + return *input +} + +// GetMode returns the given mode or the default value if it is nil +func (t *MetricsMonitorResourceMode) GetMode() MetricsMonitorResourceMode { + return MetricsMonitorResourceModeOrDefault(t, MetricsMonitorResourceModeAuto) +} diff --git a/pkg/apis/deployment/v1alpha/server_group.go b/pkg/apis/deployment/v1alpha/server_group.go index 8d75043fe..d6ceefe16 100644 --- a/pkg/apis/deployment/v1alpha/server_group.go +++ b/pkg/apis/deployment/v1alpha/server_group.go @@ -134,7 +134,7 @@ func (g ServerGroup) IsArangosync() bool { // IsExportMetrics return true when the group can be used with the arangodbexporter func (g ServerGroup) IsExportMetrics() bool { switch g { - case ServerGroupCoordinators, ServerGroupDBServers: + case ServerGroupCoordinators, ServerGroupDBServers, ServerGroupSingle: return true default: return false diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index 8dcf0dd78..bd6b1c12e 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -547,6 +547,27 @@ func (in *MetricsAuthenticationSpec) DeepCopy() *MetricsAuthenticationSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MetricsPrometheusSpec) DeepCopyInto(out *MetricsPrometheusSpec) { + *out = *in + if in.MonitorResource != nil { + in, out := &in.MonitorResource, &out.MonitorResource + *out = new(MetricsMonitorResourceMode) + **out = **in + } + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsPrometheusSpec. +func (in *MetricsPrometheusSpec) DeepCopy() *MetricsPrometheusSpec { + if in == nil { + return nil + } + out := new(MetricsPrometheusSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { *out = *in @@ -561,6 +582,7 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { **out = **in } in.Authentication.DeepCopyInto(&out.Authentication) + in.Prometheus.DeepCopyInto(&out.Prometheus) return } diff --git a/pkg/deployment/resources/monitoring.go b/pkg/deployment/resources/monitoring.go new file mode 100644 index 000000000..db6e953fe --- /dev/null +++ b/pkg/deployment/resources/monitoring.go @@ -0,0 +1,50 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// + +package resources + +import ( + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" + //papi "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" +) + +const ( + prometheusMonitoringResourceName = "arangodb-exporter" +) + +// EnsureMonitoringResources ensures that all requested external monitoring resources are created +func (r *Resources) EnsureMonitoringResources() error { + spec := r.context.GetSpec() + metrics := spec.Metrics + + if !metrics.IsEnabled() { + return nil + } + + if err := r.ensurePrometheusMonitoringResources(metrics.Prometheus); err != nil { + return err + } + + return nil +} + +func (r *Resources) ensurePrometheusMonitoringResources(p api.MetricsPrometheusSpec) error { + return nil +} From 7ece1085ea148f8073e3886d4c895d0e70a70996 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Fri, 11 Jan 2019 14:47:56 +0100 Subject: [PATCH 10/12] Added label for exporter. --- pkg/apis/deployment/v1alpha/metrics_spec.go | 5 ++++- pkg/util/k8sutil/pods.go | 1 + pkg/util/k8sutil/util.go | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index 84845b976..354ee6c52 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -157,7 +157,10 @@ func (p *MetricsPrometheusSpec) SetDefaultsFrom(source MetricsPrometheusSpec) { // Validate the given spec func (p *MetricsPrometheusSpec) Validate() error { - return p.MonitorResource.Validate() + if p.MonitorResource != nil { + return p.MonitorResource.Validate() + } + return nil } // Validate the given spec diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index e838d686a..6bf6384ed 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -538,6 +538,7 @@ func CreateArangodPod(kubecli kubernetes.Interface, developmentMode bool, deploy c.VolumeMounts = append(c.VolumeMounts, tlsKeyfileVolumeMounts()...) } p.Spec.Containers = append(p.Spec.Containers, c) + p.Labels[LabelKeyArangoExporter] = "yes" } // Add UUID init container diff --git a/pkg/util/k8sutil/util.go b/pkg/util/k8sutil/util.go index ac1273fab..0e6f94576 100644 --- a/pkg/util/k8sutil/util.go +++ b/pkg/util/k8sutil/util.go @@ -36,6 +36,8 @@ const ( LabelKeyApp = "app" // LabelKeyRole is the key of the label used to store the role of the resource in LabelKeyRole = "role" + // LabelKeyArangoExporter is the key of the label used to indicate that a exporter is present + LabelKeyArangoExporter = "arango_exporter" // AppName is the fixed value for the "app" label AppName = "arangodb" From abcae5f827e591bc29a35377f9d029dd3f25f84b Mon Sep 17 00:00:00 2001 From: lamai93 Date: Wed, 10 Apr 2019 16:51:32 +0200 Subject: [PATCH 11/12] Added allowed paths to exporter token. --- pkg/deployment/resources/secrets.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index e902b8258..392cbbc95 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -122,8 +122,9 @@ func (r *Resources) ensureExporterTokenSecret(secrets k8sutil.SecretInterface, t if _, err := secrets.Get(tokenSecretName, metav1.GetOptions{}); k8sutil.IsNotFound(err) { // Secret not found, create it claims := map[string]interface{}{ - "iss": "arangodb", - "server_id": "exporter", + "iss": "arangodb", + "server_id": "exporter", + "allowed_paths": []string{"/_admin/statistics", "/_admin/statistics-description"}, } // Create secret owner := r.context.GetAPIObject().AsOwner() From 95c80bdd24fcb876332d1aed86ddc0dc34745c37 Mon Sep 17 00:00:00 2001 From: lamai93 Date: Fri, 3 May 2019 13:45:47 +0200 Subject: [PATCH 12/12] Removed prometheus. Updated comment. --- pkg/apis/deployment/v1alpha/metrics_spec.go | 99 ------------------- .../v1alpha/zz_generated.deepcopy.go | 22 ----- pkg/deployment/resources/monitoring.go | 50 ---------- pkg/util/k8sutil/pods.go | 8 +- 4 files changed, 4 insertions(+), 175 deletions(-) delete mode 100644 pkg/deployment/resources/monitoring.go diff --git a/pkg/apis/deployment/v1alpha/metrics_spec.go b/pkg/apis/deployment/v1alpha/metrics_spec.go index 354ee6c52..41c316bb5 100644 --- a/pkg/apis/deployment/v1alpha/metrics_spec.go +++ b/pkg/apis/deployment/v1alpha/metrics_spec.go @@ -24,7 +24,6 @@ package v1alpha import ( "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - "github.com/pkg/errors" ) // MetricsAuthenticationSpec contains spec for authentication with arangodb @@ -33,44 +32,11 @@ type MetricsAuthenticationSpec struct { JWTTokenSecretName *string `json:"jwtTokenSecretName,omitempty"` } -// MetricsMonitorResourceMode specifies the type of ServiceMonitor resource to be installed -type MetricsMonitorResourceMode string - -const ( - // MetricsMonitorResourceModeAuto detects if the CRD for ServiceMonitor is available and creates the resource if so (default) - MetricsMonitorResourceModeAuto MetricsMonitorResourceMode = "Auto" - // MetricsMonitorResourceModeNone never creates the resource - MetricsMonitorResourceModeNone MetricsMonitorResourceMode = "None" - // MetricsMonitorResourceModeAlways always creates the resource and fails if the CRD is not available - MetricsMonitorResourceModeAlways MetricsMonitorResourceMode = "Always" -) - -// IsAuto returns true if mode is MetricsMonitorResourceModeAuto -func (t MetricsMonitorResourceMode) IsAuto() bool { - return t == MetricsMonitorResourceModeAuto -} - -// IsNone returns true if mode is MetricsMonitorResourceModeNone -func (t MetricsMonitorResourceMode) IsNone() bool { - return t == MetricsMonitorResourceModeNone -} - -// IsAlways returns true if mode is MetricsMonitorResourceModeAlways -func (t MetricsMonitorResourceMode) IsAlways() bool { - return t == MetricsMonitorResourceModeAlways -} - -// MetricsPrometheusSpec contains metrics for prometheus -type MetricsPrometheusSpec struct { - MonitorResource *MetricsMonitorResourceMode `json:"monitorResource,omitempty"` -} - // MetricsSpec contains spec for arangodb exporter type MetricsSpec struct { Enabled *bool `json:"enabled,omitempty"` Image *string `json:"image,omitempty"` Authentication MetricsAuthenticationSpec `json:"authentication,omitempty"` - Prometheus MetricsPrometheusSpec `json:"prometheus,omitempty"` } // IsEnabled returns whether metrics are enabled or not @@ -96,7 +62,6 @@ func (s *MetricsSpec) SetDefaults(defaultTokenName string, isAuthenticated bool) if s.GetJWTTokenSecretName() == "" { s.Authentication.JWTTokenSecretName = util.NewString(defaultTokenName) } - s.Prometheus.SetDefaults() } // GetJWTTokenSecretName returns the token secret name or empty string @@ -120,7 +85,6 @@ func (s *MetricsSpec) SetDefaultsFrom(source MetricsSpec) { if s.Authentication.JWTTokenSecretName == nil { s.Authentication.JWTTokenSecretName = util.NewStringOrNil(source.Authentication.JWTTokenSecretName) } - s.Prometheus.SetDefaultsFrom(source.Prometheus) } // Validate the given spec @@ -132,10 +96,6 @@ func (s *MetricsSpec) Validate() error { } } - if err := s.Prometheus.Validate(); err != nil { - return err - } - return nil } @@ -143,62 +103,3 @@ func (s *MetricsSpec) Validate() error { func (s MetricsSpec) ResetImmutableFields(fieldPrefix string, target *MetricsSpec) []string { return nil } - -// SetDefaults sets default values -func (p *MetricsPrometheusSpec) SetDefaults() { -} - -// SetDefaultsFrom fills unspecified fields with a value from given source spec. -func (p *MetricsPrometheusSpec) SetDefaultsFrom(source MetricsPrometheusSpec) { - if p.MonitorResource == nil { - p.MonitorResource = NewMetricsMonitorResourceModeOrNil(source.MonitorResource) - } -} - -// Validate the given spec -func (p *MetricsPrometheusSpec) Validate() error { - if p.MonitorResource != nil { - return p.MonitorResource.Validate() - } - return nil -} - -// Validate the given spec -func (t MetricsMonitorResourceMode) Validate() error { - switch t { - case MetricsMonitorResourceModeAuto, MetricsMonitorResourceModeNone, MetricsMonitorResourceModeAlways: - return nil - default: - return maskAny(errors.Wrapf(ValidationError, "Unknown monitor resource mode: '%s'", string(t))) - } -} - -// NewMetricsMonitorResourceMode returns a reference to a string with given value. -func NewMetricsMonitorResourceMode(input MetricsMonitorResourceMode) *MetricsMonitorResourceMode { - return &input -} - -// NewMetricsMonitorResourceModeOrNil returns nil if input is nil, otherwise returns a clone of the given value. -func NewMetricsMonitorResourceModeOrNil(input *MetricsMonitorResourceMode) *MetricsMonitorResourceMode { - if input == nil { - return nil - } - - return NewMetricsMonitorResourceMode(*input) -} - -// MetricsMonitorResourceModeOrDefault returns the default value (or empty string) if input is nil, otherwise returns the referenced value. -func MetricsMonitorResourceModeOrDefault(input *MetricsMonitorResourceMode, defaultValue ...MetricsMonitorResourceMode) MetricsMonitorResourceMode { - if input == nil { - if len(defaultValue) > 0 { - return defaultValue[0] - } - return "" - } - return *input -} - -// GetMode returns the given mode or the default value if it is nil -func (t *MetricsMonitorResourceMode) GetMode() MetricsMonitorResourceMode { - return MetricsMonitorResourceModeOrDefault(t, MetricsMonitorResourceModeAuto) -} diff --git a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go index aa3c924be..eec1f9beb 100644 --- a/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go +++ b/pkg/apis/deployment/v1alpha/zz_generated.deepcopy.go @@ -576,27 +576,6 @@ func (in *MetricsAuthenticationSpec) DeepCopy() *MetricsAuthenticationSpec { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *MetricsPrometheusSpec) DeepCopyInto(out *MetricsPrometheusSpec) { - *out = *in - if in.MonitorResource != nil { - in, out := &in.MonitorResource, &out.MonitorResource - *out = new(MetricsMonitorResourceMode) - **out = **in - } - return -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MetricsPrometheusSpec. -func (in *MetricsPrometheusSpec) DeepCopy() *MetricsPrometheusSpec { - if in == nil { - return nil - } - out := new(MetricsPrometheusSpec) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { *out = *in @@ -611,7 +590,6 @@ func (in *MetricsSpec) DeepCopyInto(out *MetricsSpec) { **out = **in } in.Authentication.DeepCopyInto(&out.Authentication) - in.Prometheus.DeepCopyInto(&out.Prometheus) return } diff --git a/pkg/deployment/resources/monitoring.go b/pkg/deployment/resources/monitoring.go deleted file mode 100644 index db6e953fe..000000000 --- a/pkg/deployment/resources/monitoring.go +++ /dev/null @@ -1,50 +0,0 @@ -// -// DISCLAIMER -// -// Copyright 2018 ArangoDB GmbH, Cologne, Germany -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Copyright holder is ArangoDB GmbH, Cologne, Germany -// - -package resources - -import ( - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1alpha" - //papi "github.com/coreos/prometheus-operator/pkg/apis/monitoring/v1" -) - -const ( - prometheusMonitoringResourceName = "arangodb-exporter" -) - -// EnsureMonitoringResources ensures that all requested external monitoring resources are created -func (r *Resources) EnsureMonitoringResources() error { - spec := r.context.GetSpec() - metrics := spec.Metrics - - if !metrics.IsEnabled() { - return nil - } - - if err := r.ensurePrometheusMonitoringResources(metrics.Prometheus); err != nil { - return err - } - - return nil -} - -func (r *Resources) ensurePrometheusMonitoringResources(p api.MetricsPrometheusSpec) error { - return nil -} diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 3cbd7b3e3..da52661fd 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -94,8 +94,8 @@ func IsPodReady(pod *v1.Pod) bool { return condition != nil && condition.Status == v1.ConditionTrue } -// IsPodSucceeded returns true if all containers of the pod -// have terminated with exit code 0. +// IsPodSucceeded returns true if the arangodb container of the pod +// has terminated with exit code 0. func IsPodSucceeded(pod *v1.Pod) bool { if pod.Status.Phase == v1.PodSucceeded { return true @@ -114,8 +114,8 @@ func IsPodSucceeded(pod *v1.Pod) bool { } } -// IsPodFailed returns true if all containers of the pod -// have terminated and at least one of them wih a non-zero exit code. +// IsPodFailed returns true if the arangodb container of the pod +// has terminated wih a non-zero exit code. func IsPodFailed(pod *v1.Pod) bool { if pod.Status.Phase == v1.PodFailed { return true