From 274e62cc2e2a0890bd55fea398957ac909ec8f1c Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Thu, 15 Apr 2021 09:54:10 +0200 Subject: [PATCH 1/6] Timeouts handling to ArangoD and K8s second draft third draft fourth draft rebase configurable timeouts pass context to cluster inspection pass context when chaning password for an user pass context during chaos tests add disclaimer make fmt calculate reconciliation timeout use context to arangosync access package secret remove CreateChildContext from resourcres apply context in GetPod pass context for deployment functions apply context when updating status of the deployment go mod tidy first my code review my second code review fix unit tests --- main.go | 14 +- .../arango/backup/arango_client_impl.go | 14 +- pkg/deployment/access_package.go | 38 +++-- pkg/deployment/chaos/context.go | 9 +- pkg/deployment/chaos/monkey.go | 15 +- pkg/deployment/cleanup.go | 21 ++- pkg/deployment/cluster_scaling_integration.go | 40 +++-- pkg/deployment/context_impl.go | 114 +++++++++----- pkg/deployment/deployment.go | 65 +++++--- pkg/deployment/deployment_finalizers.go | 28 ++-- pkg/deployment/deployment_inspector.go | 103 +++++++++---- pkg/deployment/deployment_pod_sync_test.go | 8 +- pkg/deployment/deployment_run_test.go | 7 +- pkg/deployment/deployment_suite_test.go | 6 +- pkg/deployment/images.go | 35 +++-- pkg/deployment/images_test.go | 4 +- pkg/deployment/members.go | 5 +- pkg/deployment/pod/encryption.go | 10 +- pkg/deployment/reconcile/action_add_member.go | 9 +- .../reconcile/action_backup_restore.go | 20 ++- .../reconcile/action_backup_restore_clean.go | 2 +- .../action_bootstrap_set_password.go | 37 +++-- .../reconcile/action_bootstrap_update.go | 2 +- .../reconcile/action_cleanout_member.go | 55 +++++-- .../action_cluster_member_cleanup.go | 22 ++- pkg/deployment/reconcile/action_context.go | 105 ++++++------- .../action_disable_scaling_cluster.go | 2 +- .../action_enable_scaling_cluster.go | 2 +- .../reconcile/action_encryption_add.go | 12 +- .../reconcile/action_encryption_propagated.go | 2 +- .../reconcile/action_encryption_refresh.go | 23 ++- .../reconcile/action_encryption_remove.go | 8 +- .../action_encryption_status_update.go | 12 +- pkg/deployment/reconcile/action_helper.go | 4 +- pkg/deployment/reconcile/action_idle.go | 2 +- pkg/deployment/reconcile/action_jwt_add.go | 8 +- pkg/deployment/reconcile/action_jwt_clean.go | 8 +- .../reconcile/action_jwt_propagated.go | 2 +- .../reconcile/action_jwt_refresh.go | 13 +- .../reconcile/action_jwt_set_active.go | 8 +- .../reconcile/action_jwt_status_update.go | 4 +- .../reconcile/action_maintenance_disable.go | 12 +- .../reconcile/action_maintenance_enable.go | 13 +- .../reconcile/action_mark_to_remove_member.go | 2 +- pkg/deployment/reconcile/action_pvc_resize.go | 8 +- .../reconcile/action_pvc_resized.go | 2 +- .../reconcile/action_recreate_member.go | 4 +- .../reconcile/action_remove_member.go | 18 ++- .../reconcile/action_resign_leadership.go | 31 ++-- .../reconcile/action_rotate_member.go | 9 +- .../reconcile/action_rotate_start_member.go | 7 +- .../reconcile/action_rotate_stop_member.go | 2 +- .../reconcile/action_set_current_image.go | 2 +- .../reconcile/action_shutdown_member.go | 2 +- .../reconcile/action_tls_ca_append.go | 8 +- .../reconcile/action_tls_ca_clean.go | 9 +- .../reconcile/action_tls_ca_renew.go | 8 +- .../reconcile/action_tls_keyfile_clean.go | 5 +- .../reconcile/action_tls_keyfile_refresh.go | 12 +- .../reconcile/action_tls_propagated.go | 2 +- .../reconcile/action_tls_sni_update.go | 12 +- .../reconcile/action_tls_status_update.go | 10 +- .../reconcile/action_upgrade_current_image.go | 2 +- .../reconcile/action_upgrade_member.go | 6 +- .../reconcile/action_wait_for_member_up.go | 21 ++- pkg/deployment/reconcile/context.go | 45 +++--- pkg/deployment/reconcile/helper_shutdown.go | 20 ++- pkg/deployment/reconcile/plan_builder.go | 2 +- .../reconcile/plan_builder_cluster.go | 18 ++- .../reconcile/plan_builder_common.go | 15 +- .../reconcile/plan_builder_context.go | 6 +- .../reconcile/plan_builder_encryption.go | 14 +- .../reconcile/plan_builder_restore.go | 4 +- .../reconcile/plan_builder_rotate_upgrade.go | 14 +- pkg/deployment/reconcile/plan_builder_test.go | 35 ++--- .../reconcile/plan_builder_tls_sni.go | 15 +- pkg/deployment/reconcile/plan_executor.go | 13 +- pkg/deployment/reconcile/reconciler.go | 16 +- pkg/deployment/resilience/context.go | 2 +- pkg/deployment/resilience/member_failure.go | 24 +-- pkg/deployment/resources/annotations.go | 141 +++++++++++------- .../resources/certificates_client_auth.go | 12 +- pkg/deployment/resources/certificates_tls.go | 20 ++- pkg/deployment/resources/context.go | 15 +- .../resources/inspector/inspector.go | 39 ++--- pkg/deployment/resources/inspector/members.go | 15 +- pkg/deployment/resources/inspector/pdbs.go | 17 ++- pkg/deployment/resources/inspector/pods.go | 17 ++- pkg/deployment/resources/inspector/pvcs.go | 17 ++- pkg/deployment/resources/inspector/sa.go | 17 ++- pkg/deployment/resources/inspector/secrets.go | 17 ++- .../resources/inspector/services.go | 17 ++- pkg/deployment/resources/inspector/sms.go | 15 +- pkg/deployment/resources/labels.go | 79 ++++++---- pkg/deployment/resources/member_cleanup.go | 28 ++-- pkg/deployment/resources/pdbs.go | 34 +++-- pkg/deployment/resources/pod_cleanup.go | 5 +- pkg/deployment/resources/pod_creator.go | 54 ++++--- pkg/deployment/resources/pod_finalizers.go | 16 +- pkg/deployment/resources/pod_inspector.go | 9 +- pkg/deployment/resources/pod_termination.go | 66 +++++--- pkg/deployment/resources/pvc_finalizers.go | 20 ++- pkg/deployment/resources/pvc_inspector.go | 7 +- pkg/deployment/resources/pvcs.go | 12 +- pkg/deployment/resources/secret_hashes.go | 8 +- pkg/deployment/resources/secrets.go | 129 ++++++++++------ pkg/deployment/resources/servicemonitor.go | 22 ++- pkg/deployment/resources/services.go | 53 +++++-- pkg/replication/server_endpoint_api.go | 4 +- pkg/replication/sync_client.go | 6 +- pkg/util/arangod/cleanout_server.go | 2 +- pkg/util/arangod/client.go | 25 +++- pkg/util/arangod/dbserver.go | 17 ++- pkg/util/k8sutil/finalizers.go | 32 +++- pkg/util/k8sutil/inspector/inspector.go | 6 +- pkg/util/k8sutil/pods.go | 7 +- pkg/util/k8sutil/pvc.go | 7 +- pkg/util/k8sutil/secrets.go | 59 +++++--- pkg/util/k8sutil/services.go | 31 ++-- pkg/util/k8sutil/util.go | 22 ++- tests/auth_test.go | 13 +- tests/sync/main.go | 4 +- tests/test_util.go | 5 +- 123 files changed, 1624 insertions(+), 865 deletions(-) diff --git a/main.go b/main.go index de07c800b..ae4bf6cdd 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package main @@ -32,6 +33,8 @@ import ( "strings" "time" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/operator/scope" "github.com/arangodb/kube-arangodb/pkg/deployment/features" @@ -114,6 +117,10 @@ var ( singleMode bool scope string } + timeouts struct { + k8s time.Duration + arangoD time.Duration + } chaosOptions struct { allowed bool } @@ -143,7 +150,8 @@ func init() { f.BoolVar(&chaosOptions.allowed, "chaos.allowed", false, "Set to allow chaos in deployments. Only activated when allowed and enabled in deployment") f.BoolVar(&operatorOptions.singleMode, "mode.single", false, "Enable single mode in Operator. WARNING: There should be only one replica of Operator, otherwise Operator can take unexpected actions") f.StringVar(&operatorOptions.scope, "scope", scope.DefaultScope.String(), "Define scope on which Operator works. Legacy - pre 1.1.0 scope with limited cluster access") - + f.DurationVar(&timeouts.k8s, "timeout.k8s", time.Second*3, "The request timeout to the kubernetes") + f.DurationVar(&timeouts.arangoD, "timeout.arangod", time.Second*10, "The request timeout to the ArangoDB") features.Init(&cmdMain) } @@ -168,6 +176,8 @@ func cmdMainRun(cmd *cobra.Command, args []string) { ip := os.Getenv(constants.EnvOperatorPodIP) deploymentApi.DefaultImage = operatorOptions.arangoImage + k8sutil.SetRequestTimeout(timeouts.k8s) + arangod.SetRequestTimeout(timeouts.arangoD) // Prepare log service var err error diff --git a/pkg/backup/handlers/arango/backup/arango_client_impl.go b/pkg/backup/handlers/arango/backup/arango_client_impl.go index 40a552b8b..8055c2397 100644 --- a/pkg/backup/handlers/arango/backup/arango_client_impl.go +++ b/pkg/backup/handlers/arango/backup/arango_client_impl.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Lars Maier +// Author Tomasz Mielech // package backup @@ -128,9 +129,10 @@ func (ac *arangoClientBackupImpl) Get(backupID driver.BackupID) (driver.BackupMe } } -func (ac *arangoClientBackupImpl) getCredentialsFromSecret(secretName string) (interface{}, error) { - - token, err := k8sutil.GetTokenSecret(ac.kubecli.CoreV1().Secrets(ac.backup.Namespace), secretName) +func (ac *arangoClientBackupImpl) getCredentialsFromSecret(ctx context.Context, secretName string) (interface{}, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + token, err := k8sutil.GetTokenSecret(ctxChild, ac.kubecli.CoreV1().Secrets(ac.backup.Namespace), secretName) + cancel() if err != nil { return nil, err } @@ -152,7 +154,7 @@ func (ac *arangoClientBackupImpl) Upload(backupID driver.BackupID) (driver.Backu return "", errors.Newf("upload was called but no upload spec was given") } - cred, err := ac.getCredentialsFromSecret(uploadSpec.CredentialsSecretName) + cred, err := ac.getCredentialsFromSecret(ctx, uploadSpec.CredentialsSecretName) if err != nil { return "", err } @@ -169,7 +171,7 @@ func (ac *arangoClientBackupImpl) Download(backupID driver.BackupID) (driver.Bac return "", errors.Newf("Download was called but not download spec was given") } - cred, err := ac.getCredentialsFromSecret(downloadSpec.CredentialsSecretName) + cred, err := ac.getCredentialsFromSecret(ctx, downloadSpec.CredentialsSecretName) if err != nil { return "", err } diff --git a/pkg/deployment/access_package.go b/pkg/deployment/access_package.go index 43c77056c..3b7e1d8c4 100644 --- a/pkg/deployment/access_package.go +++ b/pkg/deployment/access_package.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -46,7 +47,7 @@ const ( // createAccessPackages creates a arangosync access packages specified // in spec.sync.externalAccess.accessPackageSecretNames. -func (d *Deployment) createAccessPackages() error { +func (d *Deployment) createAccessPackages(ctx context.Context) error { log := d.deps.Log spec := d.apiObject.Spec secrets := d.deps.KubeCli.CoreV1().Secrets(d.GetNamespace()) @@ -60,13 +61,15 @@ func (d *Deployment) createAccessPackages() error { apNameMap := make(map[string]struct{}) for _, apSecretName := range spec.Sync.ExternalAccess.AccessPackageSecretNames { apNameMap[apSecretName] = struct{}{} - if err := d.ensureAccessPackage(apSecretName); err != nil { + if err := d.ensureAccessPackage(ctx, apSecretName); err != nil { return errors.WithStack(err) } } // Remove all access packages that we did build, but are no longer needed - secretList, err := secrets.List(context.Background(), metav1.ListOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + secretList, err := secrets.List(ctxChild, metav1.ListOptions{}) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to list secrets") return errors.WithStack(err) @@ -77,9 +80,12 @@ func (d *Deployment) createAccessPackages() error { // Secret is an access package if _, wanted := apNameMap[secret.GetName()]; !wanted { // We found an obsolete access package secret. Remove it. - if err := secrets.Delete(context.Background(), secret.GetName(), metav1.DeleteOptions{ + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := secrets.Delete(ctxChild, secret.GetName(), metav1.DeleteOptions{ Preconditions: &metav1.Preconditions{UID: &secret.UID}, - }); err != nil && !k8sutil.IsNotFound(err) { + }) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { // Not serious enough to stop everything now, just log and create an event log.Warn().Err(err).Msg("Failed to remove obsolete access package secret") d.CreateEvent(k8sutil.NewErrorEvent("Access Package cleanup failed", err, d.apiObject)) @@ -98,20 +104,28 @@ func (d *Deployment) createAccessPackages() error { // ensureAccessPackage creates an arangosync access package with given name // it is does not already exist. -func (d *Deployment) ensureAccessPackage(apSecretName string) error { +func (d *Deployment) ensureAccessPackage(ctx context.Context, apSecretName string) error { log := d.deps.Log ns := d.GetNamespace() secrets := d.deps.KubeCli.CoreV1().Secrets(ns) spec := d.apiObject.Spec - if _, err := secrets.Get(context.Background(), apSecretName, metav1.GetOptions{}); err == nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := secrets.Get(ctxChild, apSecretName, metav1.GetOptions{}) + cancel() + if err == nil { // Secret already exists return nil + } else if !k8sutil.IsNotFound(err) { + log.Debug().Err(err).Str("name", apSecretName).Msg("Failed to get arangosync access package secret") + return errors.WithStack(err) } // Fetch client authentication CA clientAuthSecretName := spec.Sync.Authentication.GetClientCASecretName() - clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(secrets, clientAuthSecretName, nil) + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(ctxChild, secrets, clientAuthSecretName, nil) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to get client-auth CA secret") return errors.WithStack(err) @@ -119,7 +133,7 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error { // Fetch TLS CA public key tlsCASecretName := spec.Sync.TLS.GetCASecretName() - tlsCACert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + tlsCACert, err := k8sutil.GetCACertficateSecret(ctx, secrets, tlsCASecretName) if err != nil { log.Debug().Err(err).Msg("Failed to get TLS CA secret") return errors.WithStack(err) @@ -205,7 +219,9 @@ func (d *Deployment) ensureAccessPackage(apSecretName string) error { } // Attach secret to owner secret.SetOwnerReferences(append(secret.GetOwnerReferences(), d.apiObject.AsOwner())) - if _, err := secrets.Create(context.Background(), secret, metav1.CreateOptions{}); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + if _, err := secrets.Create(ctxChild, secret, metav1.CreateOptions{}); err != nil { // Failed to create secret log.Debug().Err(err).Str("secret-name", apSecretName).Msg("Failed to create access package Secret") return errors.WithStack(err) diff --git a/pkg/deployment/chaos/context.go b/pkg/deployment/chaos/context.go index 72d97bd6e..ef3723253 100644 --- a/pkg/deployment/chaos/context.go +++ b/pkg/deployment/chaos/context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package chaos import ( + "context" + v1 "k8s.io/api/core/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -34,7 +37,7 @@ type Context interface { GetSpec() api.DeploymentSpec // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // GetOwnedPods returns a list of all pods owned by the deployment. - GetOwnedPods() ([]v1.Pod, error) + GetOwnedPods(ctx context.Context) ([]v1.Pod, error) } diff --git a/pkg/deployment/chaos/monkey.go b/pkg/deployment/chaos/monkey.go index e7ff95f54..329c4b26f 100644 --- a/pkg/deployment/chaos/monkey.go +++ b/pkg/deployment/chaos/monkey.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package chaos import ( + "context" "math/rand" "time" @@ -50,6 +52,9 @@ func NewMonkey(log zerolog.Logger, context Context) *Monkey { // Run the monkey until the given channel is closed. func (m Monkey) Run(stopCh <-chan struct{}) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { spec := m.context.GetSpec() if spec.Chaos.IsEnabled() { @@ -57,7 +62,7 @@ func (m Monkey) Run(stopCh <-chan struct{}) { chance := float64(spec.Chaos.GetKillPodProbability()) / 100.0 if rand.Float64() < chance { // Let's introduce pod chaos - if err := m.killRandomPod(); err != nil { + if err := m.killRandomPod(ctx); err != nil { log.Info().Err(err).Msg("Failed to kill random pod") } } @@ -74,8 +79,8 @@ func (m Monkey) Run(stopCh <-chan struct{}) { } // killRandomPod fetches all owned pods and tries to kill one. -func (m Monkey) killRandomPod() error { - pods, err := m.context.GetOwnedPods() +func (m Monkey) killRandomPod(ctx context.Context) error { + pods, err := m.context.GetOwnedPods(ctx) if err != nil { return errors.WithStack(err) } @@ -85,7 +90,7 @@ func (m Monkey) killRandomPod() error { } p := pods[rand.Intn(len(pods))] m.log.Info().Str("pod-name", p.GetName()).Msg("Killing pod") - if err := m.context.DeletePod(p.GetName()); err != nil { + if err := m.context.DeletePod(ctx, p.GetName()); err != nil { return errors.WithStack(err) } return nil diff --git a/pkg/deployment/cleanup.go b/pkg/deployment/cleanup.go index 86472357f..d23726177 100644 --- a/pkg/deployment/cleanup.go +++ b/pkg/deployment/cleanup.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -25,26 +26,30 @@ package deployment import ( "context" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" "github.com/arangodb/kube-arangodb/pkg/util" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" - core "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) // removePodFinalizers removes all finalizers from all pods owned by us. -func (d *Deployment) removePodFinalizers(cachedStatus inspectorInterface.Inspector) error { +func (d *Deployment) removePodFinalizers(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { log := d.deps.Log kubecli := d.GetKubeCli() if err := cachedStatus.IteratePods(func(pod *core.Pod) error { - if err := k8sutil.RemovePodFinalizers(log, kubecli, pod, pod.GetFinalizers(), true); err != nil { + if err := k8sutil.RemovePodFinalizers(ctx, log, kubecli, pod, pod.GetFinalizers(), true); err != nil { log.Warn().Err(err).Msg("Failed to remove pod finalizers") return err } - if err := kubecli.CoreV1().Pods(pod.GetNamespace()).Delete(context.Background(), pod.GetName(), meta.DeleteOptions{ + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if err := kubecli.CoreV1().Pods(pod.GetNamespace()).Delete(ctxChild, pod.GetName(), meta.DeleteOptions{ GracePeriodSeconds: util.NewInt64(1), }); err != nil { if !k8sutil.IsNotFound(err) { @@ -61,12 +66,12 @@ func (d *Deployment) removePodFinalizers(cachedStatus inspectorInterface.Inspect } // removePVCFinalizers removes all finalizers from all PVCs owned by us. -func (d *Deployment) removePVCFinalizers(cachedStatus inspectorInterface.Inspector) error { +func (d *Deployment) removePVCFinalizers(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { log := d.deps.Log kubecli := d.GetKubeCli() if err := cachedStatus.IteratePersistentVolumeClaims(func(pvc *core.PersistentVolumeClaim) error { - if err := k8sutil.RemovePVCFinalizers(log, kubecli, pvc, pvc.GetFinalizers(), true); err != nil { + if err := k8sutil.RemovePVCFinalizers(ctx, log, kubecli, pvc, pvc.GetFinalizers(), true); err != nil { log.Warn().Err(err).Msg("Failed to remove PVC finalizers") return err } diff --git a/pkg/deployment/cluster_scaling_integration.go b/pkg/deployment/cluster_scaling_integration.go index 42b3e2a89..4ba09f1a6 100644 --- a/pkg/deployment/cluster_scaling_integration.go +++ b/pkg/deployment/cluster_scaling_integration.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -80,14 +81,14 @@ func (ci *clusterScalingIntegration) SendUpdateToCluster(spec api.DeploymentSpec // checkScalingCluster checks if inspection // returns true if inspection occurred -func (ci *clusterScalingIntegration) checkScalingCluster(expectSuccess bool) bool { +func (ci *clusterScalingIntegration) checkScalingCluster(ctx context.Context, expectSuccess bool) bool { ci.scaleEnabled.mutex.Lock() defer ci.scaleEnabled.mutex.Unlock() if !ci.scaleEnabled.enabled { // Check if it is possible to turn on scaling without any issue status, _ := ci.depl.GetStatus() - if status.Plan.IsEmpty() && ci.setNumberOfServers() == nil { + if status.Plan.IsEmpty() && ci.setNumberOfServers(ctx) == nil { // Scaling should be enabled because there is no Plan. // It can happen when the enabling action fails ci.scaleEnabled.enabled = true @@ -100,8 +101,6 @@ func (ci *clusterScalingIntegration) checkScalingCluster(expectSuccess bool) boo } // Update cluster with our state - ctx := context.Background() - //expectSuccess := *goodInspections > 0 || time.Since(start) > maxClusterBootstrapTime safeToAskCluster, err := ci.updateClusterServerCount(ctx, expectSuccess) if err != nil { if expectSuccess { @@ -124,10 +123,13 @@ func (ci *clusterScalingIntegration) checkScalingCluster(expectSuccess bool) boo func (ci *clusterScalingIntegration) ListenForClusterEvents(stopCh <-chan struct{}) { start := time.Now() goodInspections := 0 + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + for { expectSuccess := goodInspections > 0 || time.Since(start) > maxClusterBootstrapTime - if ci.checkScalingCluster(expectSuccess) { + if ci.checkScalingCluster(ctx, expectSuccess) { goodInspections++ } @@ -144,11 +146,17 @@ func (ci *clusterScalingIntegration) ListenForClusterEvents(stopCh <-chan struct // Perform a single inspection of the cluster func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectSuccess bool) error { log := ci.log - c, err := ci.depl.clientCache.GetDatabase(ctx) + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := ci.depl.clientCache.GetDatabase(ctxChild) + cancel() if err != nil { return errors.WithStack(err) } - req, err := arangod.GetNumberOfServers(ctx, c.Connection()) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + req, err := arangod.GetNumberOfServers(ctxChild, c.Connection()) + cancel() if err != nil { if expectSuccess { log.Debug().Err(err).Msg("Failed to get number of servers") @@ -191,7 +199,9 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectS } // Let's update the spec apiObject := ci.depl.apiObject - current, err := ci.depl.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(apiObject.Namespace).Get(context.Background(), apiObject.Name, metav1.GetOptions{}) + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + current, err := ci.depl.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(apiObject.Namespace).Get(ctxChild, apiObject.Name, metav1.GetOptions{}) + cancel() if err != nil { return errors.WithStack(err) } @@ -211,7 +221,7 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectS // Restore original spec in cluster ci.SendUpdateToCluster(current.Spec) } else { - if err := ci.depl.updateCRSpec(*newSpec); err != nil { + if err := ci.depl.updateCRSpec(ctx, *newSpec); err != nil { log.Warn().Err(err).Msg("Failed to update current deployment") return errors.WithStack(err) } @@ -288,12 +298,11 @@ func (ci *clusterScalingIntegration) GetLastNumberOfServers() arangod.NumberOfSe } // DisableScalingCluster disables scaling DBservers and coordinators -func (ci *clusterScalingIntegration) DisableScalingCluster() error { +func (ci *clusterScalingIntegration) DisableScalingCluster(ctx context.Context) error { ci.scaleEnabled.mutex.Lock() defer ci.scaleEnabled.mutex.Unlock() // Turn off scaling DBservers and coordinators in arangoDB for the UI - ctx := context.Background() if err := ci.depl.SetNumberOfServers(ctx, nil, nil); err != nil { return errors.WithStack(err) } @@ -303,7 +312,7 @@ func (ci *clusterScalingIntegration) DisableScalingCluster() error { } // EnableScalingCluster enables scaling DBservers and coordinators -func (ci *clusterScalingIntegration) EnableScalingCluster() error { +func (ci *clusterScalingIntegration) EnableScalingCluster(ctx context.Context) error { ci.scaleEnabled.mutex.Lock() defer ci.scaleEnabled.mutex.Unlock() @@ -311,15 +320,14 @@ func (ci *clusterScalingIntegration) EnableScalingCluster() error { return nil } - if err := ci.setNumberOfServers(); err != nil { + if err := ci.setNumberOfServers(ctx); err != nil { return errors.WithStack(err) } ci.scaleEnabled.enabled = true return nil } -func (ci *clusterScalingIntegration) setNumberOfServers() error { - ctx := context.Background() +func (ci *clusterScalingIntegration) setNumberOfServers(ctx context.Context) error { spec := ci.depl.GetSpec() numOfCoordinators := spec.Coordinators.GetCount() numOfDBServers := spec.DBServers.GetCount() diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index 2a45dff56..c158555a0 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -66,9 +67,14 @@ import ( v1 "k8s.io/api/core/v1" ) +var _ resources.Context = &Deployment{} + // GetBackup receives information about a backup resource -func (d *Deployment) GetBackup(backup string) (*backupApi.ArangoBackup, error) { - return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(context.Background(), backup, meta.GetOptions{}) +func (d *Deployment) GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + return d.deps.DatabaseCRCli.BackupV1().ArangoBackups(d.Namespace()).Get(ctxChild, backup, meta.GetOptions{}) } // GetAPIObject returns the deployment as k8s object. @@ -147,14 +153,14 @@ func (d *Deployment) getStatus() (api.DeploymentStatus, int32) { // updates the resources in k8s. // If the given last version does not match the actual last version of the status object, // an error is returned. -func (d *Deployment) UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { +func (d *Deployment) UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error { d.status.mutex.Lock() defer d.status.mutex.Unlock() - return d.updateStatus(status, lastVersion, force...) + return d.updateStatus(ctx, status, lastVersion, force...) } -func (d *Deployment) updateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { +func (d *Deployment) updateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error { if d.status.version != lastVersion { // Status is obsolete d.deps.Log.Error(). @@ -165,14 +171,14 @@ func (d *Deployment) updateStatus(status api.DeploymentStatus, lastVersion int32 } d.status.version++ d.status.last = *status.DeepCopy() - if err := d.updateCRStatus(force...); err != nil { + if err := d.updateCRStatus(ctx, force...); err != nil { return errors.WithStack(err) } return nil } // UpdateMember updates the deployment status wrt the given member. -func (d *Deployment) UpdateMember(member api.MemberStatus) error { +func (d *Deployment) UpdateMember(ctx context.Context, member api.MemberStatus) error { status, lastVersion := d.GetStatus() _, group, found := status.Members.ElementByID(member.ID) if !found { @@ -181,7 +187,7 @@ func (d *Deployment) UpdateMember(member api.MemberStatus) error { if err := status.Members.Update(member, group); err != nil { return errors.WithStack(err) } - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { d.deps.Log.Debug().Err(err).Msg("Updating CR status failed") return errors.WithStack(err) } @@ -324,7 +330,7 @@ func (d *Deployment) GetSyncServerClient(ctx context.Context, group api.ServerGr ns := d.apiObject.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := d.apiObject.Spec.Sync.Monitoring.GetTokenSecretName() - monitoringToken, err := k8sutil.GetTokenSecret(secrets, secretName) + monitoringToken, err := k8sutil.GetTokenSecret(ctx, secrets, secretName) if err != nil { log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") return nil, errors.WithStack(err) @@ -355,7 +361,7 @@ func (d *Deployment) GetSyncServerClient(ctx context.Context, group api.ServerGr // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. -func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, error) { +func (d *Deployment) CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) { log := d.deps.Log status, lastVersion := d.GetStatus() id, err := createMember(log, &status, group, id, d.apiObject) @@ -364,7 +370,7 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err return "", errors.WithStack(err) } // Save added member - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Updating CR status failed") return "", errors.WithStack(err) } @@ -375,16 +381,22 @@ func (d *Deployment) CreateMember(group api.ServerGroup, id string) (string, err } // GetPod returns pod. -func (d *Deployment) GetPod(podName string) (*v1.Pod, error) { - return d.deps.KubeCli.CoreV1().Pods(d.GetNamespace()).Get(context.Background(), podName, meta.GetOptions{}) +func (d *Deployment) GetPod(ctx context.Context, podName string) (*v1.Pod, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + return d.deps.KubeCli.CoreV1().Pods(d.GetNamespace()).Get(ctxChild, podName, meta.GetOptions{}) } // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (d *Deployment) DeletePod(podName string) error { +func (d *Deployment) DeletePod(ctx context.Context, podName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to remove pod") return errors.WithStack(err) } @@ -393,13 +405,17 @@ func (d *Deployment) DeletePod(podName string) error { // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. -func (d *Deployment) CleanupPod(p *v1.Pod) error { +func (d *Deployment) CleanupPod(ctx context.Context, p *v1.Pod) error { log := d.deps.Log podName := p.GetName() ns := p.GetNamespace() options := meta.NewDeleteOptions(0) options.Preconditions = meta.NewUIDPreconditions(string(p.GetUID())) - if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, *options); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, *options) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to cleanup pod") return errors.WithStack(err) } @@ -408,18 +424,23 @@ func (d *Deployment) CleanupPod(p *v1.Pod) error { // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (d *Deployment) RemovePodFinalizers(podName string) error { +func (d *Deployment) RemovePodFinalizers(ctx context.Context, podName string) error { log := d.deps.Log ns := d.GetNamespace() kubecli := d.deps.KubeCli - p, err := kubecli.CoreV1().Pods(ns).Get(context.Background(), podName, meta.GetOptions{}) + + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + p, err := kubecli.CoreV1().Pods(ns).Get(ctxChild, podName, meta.GetOptions{}) + cancel() if err != nil { if k8sutil.IsNotFound(err) { return nil } return errors.WithStack(err) } - if err := k8sutil.RemovePodFinalizers(log, d.deps.KubeCli, p, p.GetFinalizers(), true); err != nil { + + err = k8sutil.RemovePodFinalizers(ctx, log, d.deps.KubeCli, p, p.GetFinalizers(), true) + if err != nil { return errors.WithStack(err) } return nil @@ -427,10 +448,13 @@ func (d *Deployment) RemovePodFinalizers(podName string) error { // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (d *Deployment) DeletePvc(pvcName string) error { +func (d *Deployment) DeletePvc(ctx context.Context, pvcName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(context.Background(), pvcName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(ctxChild, pvcName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pvc", pvcName).Msg("Failed to remove pvc") return errors.WithStack(err) } @@ -439,8 +463,11 @@ func (d *Deployment) DeletePvc(pvcName string) error { // UpdatePvc updated a persistent volume claim in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (d *Deployment) UpdatePvc(pvc *v1.PersistentVolumeClaim) error { - _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(context.Background(), pvc, meta.UpdateOptions{}) +func (d *Deployment) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(ctxChild, pvc, meta.UpdateOptions{}) if err == nil { return nil } @@ -471,8 +498,11 @@ func (d *Deployment) GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) { } // GetPvc gets a PVC by the given name, in the samespace of the deployment. -func (d *Deployment) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) { - pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(context.Background(), pvcName, meta.GetOptions{}) +func (d *Deployment) GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + pvc, err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(d.apiObject.GetNamespace()).Get(ctxChild, pvcName, meta.GetOptions{}) if err != nil { log.Debug().Err(err).Str("pvc-name", pvcName).Msg("Failed to get PVC") return nil, errors.WithStack(err) @@ -495,10 +525,13 @@ func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatu // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. -func (d *Deployment) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { +func (d *Deployment) DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error { secretName := k8sutil.CreateTLSKeyfileSecretName(d.apiObject.GetName(), group.AsRole(), member.ID) ns := d.apiObject.GetNamespace() - if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(context.Background(), secretName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(ctxChild, secretName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { return errors.WithStack(err) } return nil @@ -524,12 +557,12 @@ func (d *Deployment) InvalidateSyncStatus() { d.resources.InvalidateSyncStatus() } -func (d *Deployment) DisableScalingCluster() error { - return d.clusterScalingIntegration.DisableScalingCluster() +func (d *Deployment) DisableScalingCluster(ctx context.Context) error { + return d.clusterScalingIntegration.DisableScalingCluster(ctx) } -func (d *Deployment) EnableScalingCluster() error { - return d.clusterScalingIntegration.EnableScalingCluster() +func (d *Deployment) EnableScalingCluster(ctx context.Context) error { + return d.clusterScalingIntegration.EnableScalingCluster(ctx) } // GetAgencyPlan returns agency plan @@ -546,8 +579,8 @@ func (d *Deployment) GetAgencyData(ctx context.Context, i interface{}, keyParts return err } -func (d *Deployment) RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) { - return d.resources.RenderPodForMember(cachedStatus, spec, status, memberID, imageInfo) +func (d *Deployment) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) { + return d.resources.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) } func (d *Deployment) SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) { @@ -562,7 +595,7 @@ func (d *Deployment) GetArangoImage() string { return d.config.ArangoImage } -func (d *Deployment) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { +func (d *Deployment) WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error { d.status.mutex.Lock() defer d.status.mutex.Unlock() @@ -574,7 +607,7 @@ func (d *Deployment) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, return nil } - return d.updateStatus(status, version, force...) + return d.updateStatus(ctx, status, version, force...) } func (d *Deployment) SecretsInterface() k8sutil.SecretInterface { @@ -585,8 +618,11 @@ func (d *Deployment) GetName() string { return d.apiObject.GetName() } -func (d *Deployment) GetOwnedPods() ([]v1.Pod, error) { - pods, err := d.GetKubeCli().CoreV1().Pods(d.apiObject.GetNamespace()).List(context.Background(), meta.ListOptions{}) +func (d *Deployment) GetOwnedPods(ctx context.Context) ([]v1.Pod, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + pods, err := d.GetKubeCli().CoreV1().Pods(d.apiObject.GetNamespace()).List(ctxChild, meta.ListOptions{}) if err != nil { return nil, err } diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index 7ae06a836..dadcd4235 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -222,24 +223,24 @@ func (d *Deployment) run() { if d.GetPhase() == api.DeploymentPhaseNone { // Create service monitor if d.haveServiceMonitorCRD { - if err := d.resources.EnsureServiceMonitor(); err != nil { + if err := d.resources.EnsureServiceMonitor(context.TODO()); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create service monitor", err, d.GetAPIObject())) } } // Create members - if err := d.createInitialMembers(d.apiObject); err != nil { + if err := d.createInitialMembers(context.TODO(), d.apiObject); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create initial members", err, d.GetAPIObject())) } // Create Pod Disruption Budgets - if err := d.resources.EnsurePDBs(); err != nil { + if err := d.resources.EnsurePDBs(context.TODO()); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to create pdbs", err, d.GetAPIObject())) } status, lastVersion := d.GetStatus() status.Phase = api.DeploymentPhaseRunning - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(context.TODO(), status, lastVersion); err != nil { log.Warn().Err(err).Msg("update initial CR status failed") } log.Info().Msg("start running...") @@ -257,10 +258,10 @@ func (d *Deployment) run() { } // Remove finalizers from created resources log.Info().Msg("Deployment removed, removing finalizers to prevent orphaned resources") - if err := d.removePodFinalizers(cachedStatus); err != nil { + if err := d.removePodFinalizers(context.TODO(), cachedStatus); err != nil { log.Warn().Err(err).Msg("Failed to remove Pod finalizers") } - if err := d.removePVCFinalizers(cachedStatus); err != nil { + if err := d.removePVCFinalizers(context.TODO(), cachedStatus); err != nil { log.Warn().Err(err).Msg("Failed to remove PVC finalizers") } // We're being stopped. @@ -284,7 +285,7 @@ func (d *Deployment) run() { d.lookForServiceMonitorCRD() case <-d.updateDeploymentTrigger.Done(): inspectionInterval = minInspectionInterval - if err := d.handleArangoDeploymentUpdatedEvent(); err != nil { + if err := d.handleArangoDeploymentUpdatedEvent(context.TODO()); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Failed to handle deployment update", err, d.GetAPIObject())) } @@ -298,11 +299,13 @@ func (d *Deployment) run() { } // handleArangoDeploymentUpdatedEvent is called when the deployment is updated by the user. -func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { +func (d *Deployment) handleArangoDeploymentUpdatedEvent(ctx context.Context) error { log := d.deps.Log.With().Str("deployment", d.apiObject.GetName()).Logger() // Get the most recent version of the deployment from the API server - current, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(context.Background(), d.apiObject.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + current, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctxChild, d.apiObject.GetName(), metav1.GetOptions{}) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to get current version of deployment from API server") if k8sutil.IsNotFound(err) { @@ -328,7 +331,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { if err := newAPIObject.Spec.Validate(); err != nil { d.CreateEvent(k8sutil.NewErrorEvent("Validation failed", err, d.apiObject)) // Try to reset object - if err := d.updateCRSpec(d.apiObject.Spec, true); err != nil { + if err := d.updateCRSpec(ctx, d.apiObject.Spec, true); err != nil { log.Error().Err(err).Msg("Restore original spec failed") d.CreateEvent(k8sutil.NewErrorEvent("Restore original failed", err, d.apiObject)) } @@ -342,7 +345,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { } // Save updated spec - if err := d.updateCRSpec(newAPIObject.Spec, true); err != nil { + if err := d.updateCRSpec(ctx, newAPIObject.Spec, true); err != nil { return errors.WithStack(errors.Newf("failed to update ArangoDeployment spec: %v", err)) } // Save updated accepted spec @@ -354,7 +357,7 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent() error { status.ForceStatusReload = nil } status.AcceptedSpec = newAPIObject.Spec.DeepCopy() - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(errors.Newf("failed to update ArangoDeployment status: %v", err)) } } @@ -377,7 +380,7 @@ func (d *Deployment) CreateEvent(evt *k8sutil.Event) { } // Update the status of the API object from the internal status -func (d *Deployment) updateCRStatus(force ...bool) error { +func (d *Deployment) updateCRStatus(ctx context.Context, force ...bool) error { if len(force) == 0 || !force[0] { if d.apiObject.Status.Equal(d.status.last) { // Nothing has changed @@ -396,7 +399,9 @@ func (d *Deployment) updateCRStatus(force ...bool) error { if update.GetDeletionTimestamp() == nil { ensureFinalizers(update) } - newAPIObject, err := depls.Update(context.Background(), update, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + newAPIObject, err := depls.Update(ctxChild, update, metav1.UpdateOptions{}) + cancel() if err == nil { // Update internal object d.apiObject = newAPIObject @@ -406,7 +411,10 @@ func (d *Deployment) updateCRStatus(force ...bool) error { // API object may have been changed already, // Reload api object and try again var current *api.ArangoDeployment - current, err = depls.Get(context.Background(), update.GetName(), metav1.GetOptions{}) + + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + current, err = depls.Get(ctxChild, update.GetName(), metav1.GetOptions{}) + cancel() if err == nil { update = current.DeepCopy() continue @@ -422,7 +430,7 @@ func (d *Deployment) updateCRStatus(force ...bool) error { // Update the spec part of the API object (d.apiObject) // to the given object, while preserving the status. // On success, d.apiObject is updated. -func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec, force ...bool) error { +func (d *Deployment) updateCRSpec(ctx context.Context, newSpec api.DeploymentSpec, force ...bool) error { if len(force) == 0 || !force[0] { if d.apiObject.Spec.Equal(&newSpec) { @@ -440,7 +448,9 @@ func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec, force ...bool) err update.Spec = newSpec update.Status = d.status.last ns := d.apiObject.GetNamespace() - newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Update(context.Background(), update, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Update(ctxChild, update, metav1.UpdateOptions{}) + cancel() if err == nil { // Update internal object d.apiObject = newAPIObject @@ -450,7 +460,10 @@ func (d *Deployment) updateCRSpec(newSpec api.DeploymentSpec, force ...bool) err // API object may have been changed already, // Reload api object and try again var current *api.ArangoDeployment - current, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Get(context.Background(), update.GetName(), metav1.GetOptions{}) + + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + current, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Get(ctxChild, update.GetName(), metav1.GetOptions{}) + cancel() if err == nil { update = current.DeepCopy() continue @@ -504,12 +517,16 @@ func (d *Deployment) lookForServiceMonitorCRD() { // SetNumberOfServers adjust number of DBservers and coordinators in arangod func (d *Deployment) SetNumberOfServers(ctx context.Context, noCoordinators, noDBServers *int) error { - c, err := d.clientCache.GetDatabase(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := d.clientCache.GetDatabase(ctxChild) + cancel() if err != nil { return errors.WithStack(err) } - err = arangod.SetNumberOfServers(ctx, c.Connection(), noCoordinators, noDBServers) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + err = arangod.SetNumberOfServers(ctxChild, c.Connection(), noCoordinators, noDBServers) + cancel() if err != nil { return errors.WithStack(err) } @@ -520,7 +537,7 @@ func (d *Deployment) getArangoDeployment() *api.ArangoDeployment { return d.apiObject } -func (d *Deployment) ApplyPatch(p ...patch.Item) error { +func (d *Deployment) ApplyPatch(ctx context.Context, p ...patch.Item) error { parser := patch.Patch(p) data, err := parser.Marshal() @@ -530,7 +547,9 @@ func (d *Deployment) ApplyPatch(p ...patch.Item) error { c := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()) - depl, err := c.Patch(context.Background(), d.apiObject.GetName(), types.JSONPatchType, data, metav1.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + depl, err := c.Patch(ctxChild, d.apiObject.GetName(), types.JSONPatchType, data, metav1.PatchOptions{}) + cancel() if err != nil { return err } diff --git a/pkg/deployment/deployment_finalizers.go b/pkg/deployment/deployment_finalizers.go index 62f4b84e6..8e2375e1a 100644 --- a/pkg/deployment/deployment_finalizers.go +++ b/pkg/deployment/deployment_finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -56,7 +57,9 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i var removalList []string depls := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.GetNamespace()) - updated, err := depls.Get(context.Background(), d.apiObject.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + updated, err := depls.Get(ctxChild, d.apiObject.GetName(), metav1.GetOptions{}) + cancel() if err != nil { return errors.WithStack(err) } @@ -73,7 +76,7 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i } // Remove finalizers (if needed) if len(removalList) > 0 { - if err := removeDeploymentFinalizers(log, d.deps.DatabaseCRCli, updated, removalList); err != nil { + if err := removeDeploymentFinalizers(ctx, log, d.deps.DatabaseCRCli, updated, removalList); err != nil { log.Debug().Err(err).Msg("Failed to update ArangoDeployment (to remove finalizers)") return errors.WithStack(err) } @@ -83,11 +86,11 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i // inspectRemoveChildFinalizers checks the finalizer condition for remove-child-finalizers. // It returns nil if the finalizer can be removed. -func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerolog.Logger, depl *api.ArangoDeployment, cachedStatus inspectorInterface.Inspector) error { - if err := d.removePodFinalizers(cachedStatus); err != nil { +func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, _ zerolog.Logger, _ *api.ArangoDeployment, cachedStatus inspectorInterface.Inspector) error { + if err := d.removePodFinalizers(ctx, cachedStatus); err != nil { return errors.WithStack(err) } - if err := d.removePVCFinalizers(cachedStatus); err != nil { + if err := d.removePVCFinalizers(ctx, cachedStatus); err != nil { return errors.WithStack(err) } @@ -95,10 +98,14 @@ func (d *Deployment) inspectRemoveChildFinalizers(ctx context.Context, log zerol } // removeDeploymentFinalizers removes the given finalizers from the given PVC. -func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, depl *api.ArangoDeployment, finalizers []string) error { +func removeDeploymentFinalizers(ctx context.Context, log zerolog.Logger, cli versioned.Interface, + depl *api.ArangoDeployment, finalizers []string) error { depls := cli.DatabaseV1().ArangoDeployments(depl.GetNamespace()) getFunc := func() (metav1.Object, error) { - result, err := depls.Get(context.Background(), depl.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + result, err := depls.Get(ctxChild, depl.GetName(), metav1.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } @@ -106,7 +113,10 @@ func removeDeploymentFinalizers(log zerolog.Logger, cli versioned.Interface, dep } updateFunc := func(updated metav1.Object) error { updatedDepl := updated.(*api.ArangoDeployment) - result, err := depls.Update(context.Background(), updatedDepl, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + result, err := depls.Update(ctxChild, updatedDepl, metav1.UpdateOptions{}) if err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 3f6e3ee72..e8092a90e 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -45,10 +46,33 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +const ( + twentySeconds = time.Second * 20 +) + var ( inspectDeploymentDurationGauges = metrics.MustRegisterGaugeVec(metricsComponent, "inspect_deployment_duration", "Amount of time taken by a single inspection of a deployment (in sec)", metrics.DeploymentName) ) +// getReconciliationTimeout gets timeout for the reconciliation loop. +// The whole reconciliation loop timeout depends on the number of nodes but not less then one minute. +func (d *Deployment) getReconciliationTimeout() (time.Duration, error) { + ctx, cancel := context.WithTimeout(context.TODO(), k8sutil.GetRequestTimeout()) + defer cancel() + + nodes, err := d.GetKubeCli().CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + return 0, errors.Wrapf(err, "Unable to get nodes") + } + + if timeout := twentySeconds * time.Duration(len(nodes.Items)); timeout > time.Minute { + return timeout, nil + } + + // The minimum timeout for the reconciliation loop. + return time.Minute, nil +} + // inspectDeployment inspects the entire deployment, creates // a plan to update if needed and inspects underlying resources. // This function should be called when: @@ -59,13 +83,22 @@ var ( func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval { log := d.deps.Log start := time.Now() + + timeout, err := d.getReconciliationTimeout() + if err != nil { + log.Error().Err(err).Msg("Unable to get nodes") + return minInspectionInterval // Retry ASAP + } + + ctxReconciliation, cancelReconciliation := context.WithTimeout(context.Background(), timeout) defer func() { + cancelReconciliation() d.deps.Log.Info().Msgf("Inspect loop took %s", time.Since(start)) }() nextInterval := lastInterval hasError := false - ctx := context.Background() + deploymentName := d.apiObject.GetName() defer metrics.SetDuration(inspectDeploymentDurationGauges.WithLabelValues(deploymentName), start) @@ -76,7 +109,9 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval } // Check deployment still exists - updated, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(context.Background(), deploymentName, metav1.GetOptions{}) + ctx, cancel := context.WithTimeout(ctxReconciliation, k8sutil.GetRequestTimeout()) + updated, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctx, deploymentName, metav1.GetOptions{}) + cancel() if k8sutil.IsNotFound(err) { // Deployment is gone log.Info().Msg("Deployment is gone") @@ -84,7 +119,7 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval return nextInterval } else if updated != nil && updated.GetDeletionTimestamp() != nil { // Deployment is marked for deletion - if err := d.runDeploymentFinalizers(ctx, cachedStatus); err != nil { + if err := d.runDeploymentFinalizers(ctxReconciliation, cachedStatus); err != nil { hasError = true d.CreateEvent(k8sutil.NewErrorEvent("ArangoDeployment finalizer inspection failed", err, d.apiObject)) } @@ -105,7 +140,8 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval d.apiObject = updated - if inspectNextInterval, err := d.inspectDeploymentWithError(ctx, nextInterval, cachedStatus); err != nil { + inspectNextInterval, err := d.inspectDeploymentWithError(ctxReconciliation, nextInterval, cachedStatus) + if err != nil { if !operatorErrors.IsReconcile(err) { nextInterval = inspectNextInterval hasError = true @@ -129,7 +165,8 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval return nextInterval.ReduceTo(maxInspectionInterval) } -func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterval util.Interval, cachedStatus inspectorInterface.Inspector) (nextInterval util.Interval, inspectError error) { +func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterval util.Interval, + cachedStatus inspectorInterface.Inspector) (nextInterval util.Interval, inspectError error) { t := time.Now() d.SetCachedStatus(cachedStatus) @@ -152,7 +189,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } else { condition, exists := status.Conditions.Get(api.ConditionTypeUpToDate) if checksum != status.AppliedVersion && (!exists || condition.IsTrue()) { - if err = d.updateCondition(api.ConditionTypeUpToDate, false, "Spec Changed", "Spec Object changed. Waiting until plan will be applied"); err != nil { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Spec Changed", "Spec Object changed. Waiting until plan will be applied"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -161,26 +198,26 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Cleanup terminated pods on the beginning of loop - if x, err := d.resources.CleanupTerminatedPods(cachedStatus); err != nil { + if x, err := d.resources.CleanupTerminatedPods(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod cleanup failed") } else { nextInterval = nextInterval.ReduceTo(x) } - if err := d.resources.EnsureArangoMembers(cachedStatus); err != nil { + if err := d.resources.EnsureArangoMembers(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "ArangoMember creation failed") } - if err := d.resources.EnsureServices(cachedStatus); err != nil { + if err := d.resources.EnsureServices(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Service creation failed") } - if err := d.resources.EnsureSecrets(d.deps.Log, cachedStatus); err != nil { + if err := d.resources.EnsureSecrets(ctx, d.deps.Log, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Secret creation failed") } // Inspect secret hashes - if err := d.resources.ValidateSecretHashes(cachedStatus); err != nil { + if err := d.resources.ValidateSecretHashes(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Secret hash validation failed") } @@ -195,7 +232,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Ensure we have image info - if retrySoon, exists, err := d.ensureImages(d.apiObject); err != nil { + if retrySoon, exists, err := d.ensureImages(ctx, d.apiObject); err != nil { return minInspectionInterval, errors.Wrapf(err, "Image detection failed") } else if retrySoon || !exists { return minInspectionInterval, nil @@ -215,16 +252,16 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Check members for resilience - if err := d.resilience.CheckMemberFailure(); err != nil { + if err := d.resilience.CheckMemberFailure(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Member failure detection failed") } // Immediate actions - if err := d.reconciler.CheckDeployment(); err != nil { + if err := d.reconciler.CheckDeployment(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Reconciler immediate actions failed") } - if interval, err := d.ensureResources(nextInterval, cachedStatus); err != nil { + if interval, err := d.ensureResources(ctx, nextInterval, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Reconciler resource recreation failed") } else { nextInterval = interval @@ -232,11 +269,11 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva // Create scale/update plan if _, ok := d.apiObject.Annotations[deployment.ArangoDeploymentPlanCleanAnnotation]; ok { - if err := d.ApplyPatch(patch.ItemRemove(patch.NewPath("metadata", "annotations", deployment.ArangoDeploymentPlanCleanAnnotation))); err != nil { + if err := d.ApplyPatch(ctx, patch.ItemRemove(patch.NewPath("metadata", "annotations", deployment.ArangoDeploymentPlanCleanAnnotation))); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to create remove annotation patch") } - if err := d.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = nil return true }, true); err != nil { @@ -249,7 +286,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } if d.apiObject.Status.Plan.IsEmpty() && status.AppliedVersion != checksum { - if err := d.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.AppliedVersion = checksum return true }); err != nil { @@ -259,7 +296,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return minInspectionInterval, nil } else if status.AppliedVersion == checksum { if !status.Plan.IsEmpty() && status.Conditions.IsTrue(api.ConditionTypeUpToDate) { - if err = d.updateCondition(api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, false, "Plan is not empty", "There are pending operations in plan"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -267,7 +304,7 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } if status.Plan.IsEmpty() && !status.Conditions.IsTrue(api.ConditionTypeUpToDate) { - if err = d.updateCondition(api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil { + if err = d.updateCondition(ctx, api.ConditionTypeUpToDate, true, "Spec is Up To Date", "Spec is Up To Date"); err != nil { return minInspectionInterval, errors.Wrapf(err, "Unable to update UpToDate condition") } @@ -285,17 +322,17 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva } // Create access packages - if err := d.createAccessPackages(); err != nil { + if err := d.createAccessPackages(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "AccessPackage creation failed") } // Inspect deployment for obsolete members - if err := d.resources.CleanupRemovedMembers(); err != nil { + if err := d.resources.CleanupRemovedMembers(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Removed member cleanup failed") } // At the end of the inspect, we cleanup terminated pods. - if x, err := d.resources.CleanupTerminatedPods(cachedStatus); err != nil { + if x, err := d.resources.CleanupTerminatedPods(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod cleanup failed") } else { nextInterval = nextInterval.ReduceTo(x) @@ -304,31 +341,31 @@ func (d *Deployment) inspectDeploymentWithError(ctx context.Context, lastInterva return } -func (d *Deployment) ensureResources(lastInterval util.Interval, cachedStatus inspectorInterface.Inspector) (util.Interval, error) { +func (d *Deployment) ensureResources(ctx context.Context, lastInterval util.Interval, cachedStatus inspectorInterface.Inspector) (util.Interval, error) { // Ensure all resources are created if d.haveServiceMonitorCRD { - if err := d.resources.EnsureServiceMonitor(); err != nil { + if err := d.resources.EnsureServiceMonitor(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "Service monitor creation failed") } } - if err := d.resources.EnsurePVCs(cachedStatus); err != nil { + if err := d.resources.EnsurePVCs(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "PVC creation failed") } - if err := d.resources.EnsurePods(cachedStatus); err != nil { + if err := d.resources.EnsurePods(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Pod creation failed") } - if err := d.resources.EnsurePDBs(); err != nil { + if err := d.resources.EnsurePDBs(ctx); err != nil { return minInspectionInterval, errors.Wrapf(err, "PDB creation failed") } - if err := d.resources.EnsureAnnotations(cachedStatus); err != nil { + if err := d.resources.EnsureAnnotations(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Annotation update failed") } - if err := d.resources.EnsureLabels(cachedStatus); err != nil { + if err := d.resources.EnsureLabels(ctx, cachedStatus); err != nil { return minInspectionInterval, errors.Wrapf(err, "Labels update failed") } @@ -345,9 +382,9 @@ func (d *Deployment) triggerCRDInspection() { d.inspectCRDTrigger.Trigger() } -func (d *Deployment) updateCondition(conditionType api.ConditionType, status bool, reason, message string) error { +func (d *Deployment) updateCondition(ctx context.Context, conditionType api.ConditionType, status bool, reason, message string) error { d.deps.Log.Info().Str("condition", string(conditionType)).Bool("status", status).Str("reason", reason).Str("message", message).Msg("Updated condition") - if err := d.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := d.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { return s.Conditions.Update(conditionType, status, reason, message) }); err != nil { return errors.Wrapf(err, "Unable to update condition") diff --git a/pkg/deployment/deployment_pod_sync_test.go b/pkg/deployment/deployment_pod_sync_test.go index 5b3ed53bb..f63c8bb64 100644 --- a/pkg/deployment/deployment_pod_sync_test.go +++ b/pkg/deployment/deployment_pod_sync_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -227,7 +227,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) { testCase.createTestPodData(deployment, api.ServerGroupSyncMasters, firstSyncMaster) name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() - auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + auth, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( @@ -306,7 +306,7 @@ func TestEnsurePod_Sync_Master(t *testing.T) { testCase.createTestPodData(deployment, api.ServerGroupSyncMasters, firstSyncMaster) name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() - auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + auth, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( @@ -408,7 +408,7 @@ func TestEnsurePod_Sync_Worker(t *testing.T) { testCase.createTestPodData(deployment, api.ServerGroupSyncWorkers, firstSyncWorker) name := testCase.ArangoDeployment.Spec.Sync.Monitoring.GetTokenSecretName() - auth, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + auth, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) require.NoError(t, err) testCase.ExpectedPod.Spec.Containers[0].LivenessProbe = createTestLivenessProbe( diff --git a/pkg/deployment/deployment_run_test.go b/pkg/deployment/deployment_run_test.go index f205b0c9a..e273af240 100644 --- a/pkg/deployment/deployment_run_test.go +++ b/pkg/deployment/deployment_run_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package deployment @@ -66,7 +67,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { for { cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) require.NoError(t, err) - err = d.resources.EnsureSecrets(log.Logger, cache) + err = d.resources.EnsureSecrets(context.Background(), log.Logger, cache) if err == nil { break } @@ -143,7 +144,7 @@ func runTestCase(t *testing.T, testCase testCaseStruct) { // Act cache, err := inspector.NewInspector(d.GetKubeCli(), d.GetMonitoringV1Cli(), d.GetArangoCli(), d.GetNamespace()) require.NoError(t, err) - err = d.resources.EnsurePods(cache) + err = d.resources.EnsurePods(context.Background(), cache) // Assert if testCase.ExpectedError != nil { diff --git a/pkg/deployment/deployment_suite_test.go b/pkg/deployment/deployment_suite_test.go index 07b288cdb..183729a26 100644 --- a/pkg/deployment/deployment_suite_test.go +++ b/pkg/deployment/deployment_suite_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package deployment import ( + "context" "fmt" "io/ioutil" "os" @@ -102,7 +104,7 @@ func createTestLifecycle() *core.Lifecycle { func createTestToken(deployment *Deployment, testCase *testCaseStruct, paths []string) (string, error) { name := testCase.ArangoDeployment.Spec.Authentication.GetJWTSecretName() - s, err := k8sutil.GetTokenSecret(deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) + s, err := k8sutil.GetTokenSecret(context.Background(), deployment.GetKubeCli().CoreV1().Secrets(testNamespace), name) if err != nil { return "", err } diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index a0463d273..a29533467 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package deployment @@ -80,7 +81,7 @@ type imagesBuilder struct { // ensureImages creates pods needed to detect ImageID for specified images. // Returns: retrySoon, error -func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, bool, error) { +func (d *Deployment) ensureImages(ctx context.Context, apiObject *api.ArangoDeployment) (bool, bool, error) { status, lastVersion := d.GetStatus() ib := imagesBuilder{ APIObject: apiObject, @@ -89,13 +90,13 @@ func (d *Deployment) ensureImages(apiObject *api.ArangoDeployment) (bool, bool, Log: d.deps.Log, KubeCli: d.deps.KubeCli, UpdateCRStatus: func(status api.DeploymentStatus) error { - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } return nil }, } - ctx := context.Background() + retrySoon, exists, err := ib.Run(ctx) if err != nil { return retrySoon, exists, errors.WithStack(err) @@ -134,12 +135,18 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima Logger() // Check if pod exists - if pod, err := ib.KubeCli.CoreV1().Pods(ns).Get(context.Background(), podName, metav1.GetOptions{}); err == nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + pod, err := ib.KubeCli.CoreV1().Pods(ns).Get(ctxChild, podName, metav1.GetOptions{}) + cancel() + if err == nil { // Pod found if k8sutil.IsPodFailed(pod) { // Wait some time before deleting the pod if time.Now().After(pod.GetCreationTimestamp().Add(30 * time.Second)) { - if err := ib.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete Image ID Pod") return false, nil } @@ -167,7 +174,9 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima log.Warn().Err(err).Msg("Failed to create Image ID Pod client") return true, nil } - v, err := client.Version(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + v, err := client.Version(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch version from Image ID Pod") return true, nil @@ -176,7 +185,10 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima enterprise := strings.ToLower(v.License) == "enterprise" // We have all the info we need now, kill the pod and store the image info. - if err := ib.KubeCli.CoreV1().Pods(ns).Delete(context.Background(), podName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err = ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete Image ID Pod") return true, nil } @@ -213,13 +225,16 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima apiObject: ib.APIObject, } - pod, err := resources.RenderArangoPod(ib.APIObject, role, id, podName, args, &imagePod) + pod, err = resources.RenderArangoPod(ib.APIObject, role, id, podName, args, &imagePod) if err != nil { log.Debug().Err(err).Msg("Failed to render image ID pod") return true, errors.WithStack(err) } - if _, err := resources.CreateArangoPod(ib.KubeCli, ib.APIObject, ib.Spec, api.ServerGroupImageDiscovery, pod); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if _, err := resources.CreateArangoPod(ctxChild, ib.KubeCli, ib.APIObject, ib.Spec, api.ServerGroupImageDiscovery, pod); err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, errors.WithStack(err) } diff --git a/pkg/deployment/images_test.go b/pkg/deployment/images_test.go index 85e4666e7..6b6763a6c 100644 --- a/pkg/deployment/images_test.go +++ b/pkg/deployment/images_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -331,7 +331,7 @@ func TestEnsureImages(t *testing.T) { require.NoError(t, err) // Act - retrySoon, _, err := d.ensureImages(d.apiObject) + retrySoon, _, err := d.ensureImages(context.Background(), d.apiObject) // Assert assert.EqualValues(t, testCase.RetrySoon, retrySoon) diff --git a/pkg/deployment/members.go b/pkg/deployment/members.go index 982c699d0..5cf36eb92 100644 --- a/pkg/deployment/members.go +++ b/pkg/deployment/members.go @@ -23,6 +23,7 @@ package deployment import ( + "context" "strings" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -37,7 +38,7 @@ import ( // createInitialMembers creates all members needed for the initial state of the deployment. // Note: This does not create any pods of PVCs -func (d *Deployment) createInitialMembers(apiObject *api.ArangoDeployment) error { +func (d *Deployment) createInitialMembers(ctx context.Context, apiObject *api.ArangoDeployment) error { log := d.deps.Log log.Debug().Msg("creating initial members...") @@ -59,7 +60,7 @@ func (d *Deployment) createInitialMembers(apiObject *api.ArangoDeployment) error // Save status log.Debug().Msg("saving initial members...") - if err := d.UpdateStatus(status, lastVersion); err != nil { + if err := d.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } // Save events diff --git a/pkg/deployment/pod/encryption.go b/pkg/deployment/pod/encryption.go index 0cf4c8eff..d5fd9d11c 100644 --- a/pkg/deployment/pod/encryption.go +++ b/pkg/deployment/pod/encryption.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package pod @@ -67,8 +68,11 @@ func GroupEncryptionSupported(mode api.DeploymentMode, group api.ServerGroup) bo } } -func GetEncryptionKey(secrets k8sutil.SecretInterface, name string) (string, []byte, bool, error) { - keyfile, err := secrets.Get(context.Background(), name, meta.GetOptions{}) +func GetEncryptionKey(ctx context.Context, secrets k8sutil.SecretInterface, name string) (string, []byte, bool, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + keyfile, err := secrets.Get(ctxChild, name, meta.GetOptions{}) if err != nil { if k8sutil.IsNotFound(err) { return "", nil, false, nil diff --git a/pkg/deployment/reconcile/action_add_member.go b/pkg/deployment/reconcile/action_add_member.go index 28e9fbdb1..29a6bdd6a 100644 --- a/pkg/deployment/reconcile/action_add_member.go +++ b/pkg/deployment/reconcile/action_add_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -64,7 +65,7 @@ type actionAddMember struct { // Returns true if the action is completely finished, false in case // the start time needs to be recorded and a ready condition needs to be checked. func (a *actionAddMember) Start(ctx context.Context) (bool, error) { - newID, err := a.actionCtx.CreateMember(a.action.Group, a.action.MemberID) + newID, err := a.actionCtx.CreateMember(ctx, a.action.Group, a.action.MemberID) if err != nil { log.Debug().Err(err).Msg("Failed to create member") return false, errors.WithStack(err) @@ -72,14 +73,14 @@ func (a *actionAddMember) Start(ctx context.Context) (bool, error) { a.newMemberID = newID if _, ok := a.action.Params[api.ActionTypeWaitForMemberUp.String()]; ok { - a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = append(s.Plan, api.NewAction(api.ActionTypeWaitForMemberInSync, a.action.Group, newID, "Wait for member in sync after creation")) return true }) } if _, ok := a.action.Params[api.ActionTypeWaitForMemberInSync.String()]; ok { - a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = append(s.Plan, api.NewAction(api.ActionTypeWaitForMemberInSync, a.action.Group, newID, "Wait for member in sync after creation")) return true }) diff --git a/pkg/deployment/reconcile/action_backup_restore.go b/pkg/deployment/reconcile/action_backup_restore.go index b9145ea75..aff976de4 100644 --- a/pkg/deployment/reconcile/action_backup_restore.go +++ b/pkg/deployment/reconcile/action_backup_restore.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,9 +27,10 @@ import ( "context" "github.com/arangodb/go-driver" + "github.com/rs/zerolog" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" - "github.com/rs/zerolog" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" ) func init() { @@ -64,12 +66,14 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return true, nil } - dbc, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + dbc, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { return false, err } - backupResource, err := a.actionCtx.GetBackup(*spec.RestoreFrom) + backupResource, err := a.actionCtx.GetBackup(ctx, *spec.RestoreFrom) if err != nil { a.log.Error().Err(err).Msg("Unable to find backup") return true, nil @@ -80,7 +84,7 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return true, nil } - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { result := &api.DeploymentRestoreResult{ RequestedFrom: spec.GetRestoreFrom(), } @@ -94,12 +98,14 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return false, err } - restoreError := dbc.Backup().Restore(ctx, driver.BackupID(backupResource.Status.Backup.ID), nil) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + restoreError := dbc.Backup().Restore(ctxChild, driver.BackupID(backupResource.Status.Backup.ID), nil) + cancel() if restoreError != nil { a.log.Error().Err(restoreError).Msg("Restore failed") } - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { result := &api.DeploymentRestoreResult{ RequestedFrom: spec.GetRestoreFrom(), } diff --git a/pkg/deployment/reconcile/action_backup_restore_clean.go b/pkg/deployment/reconcile/action_backup_restore_clean.go index 3958a2968..74a26a2e7 100644 --- a/pkg/deployment/reconcile/action_backup_restore_clean.go +++ b/pkg/deployment/reconcile/action_backup_restore_clean.go @@ -50,7 +50,7 @@ type actionBackupRestoreClean struct { } func (a actionBackupRestoreClean) Start(ctx context.Context) (bool, error) { - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Restore == nil { return false } diff --git a/pkg/deployment/reconcile/action_bootstrap_set_password.go b/pkg/deployment/reconcile/action_bootstrap_set_password.go index 4847c9b0c..e50ea7d4b 100644 --- a/pkg/deployment/reconcile/action_bootstrap_set_password.go +++ b/pkg/deployment/reconcile/action_bootstrap_set_password.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,8 @@ import ( "crypto/rand" "encoding/hex" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/go-driver" @@ -66,14 +69,15 @@ func (a actionBootstrapSetPassword) Start(ctx context.Context) (bool, error) { a.log.Warn().Msgf("User does not exist in password hashes") return true, nil } else { - ctx, c := context.WithTimeout(context.Background(), a.Timeout(spec)) - defer c() - if password, err := a.setUserPassword(ctx, user, secret.Get()); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, a.Timeout(spec)) + defer cancel() + + if password, err := a.setUserPassword(ctxChild, user, secret.Get()); err != nil { return false, err } else { passwordSha := util.SHA256FromString(password) - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.SecretHashes == nil { s.SecretHashes = &api.SecretHashes{} } @@ -99,22 +103,30 @@ func (a actionBootstrapSetPassword) Start(ctx context.Context) (bool, error) { func (a actionBootstrapSetPassword) setUserPassword(ctx context.Context, user, secret string) (string, error) { a.log.Debug().Msgf("Bootstrapping user %s, secret %s", user, secret) - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { return "", errors.WithStack(err) } - password, err := a.ensureUserPasswordSecret(user, secret) + password, err := a.ensureUserPasswordSecret(ctx, user, secret) if err != nil { return "", errors.WithStack(err) } // Obtain the user - if u, err := client.User(context.Background(), user); driver.IsNotFound(err) { - _, err := client.CreateUser(context.Background(), user, &driver.UserOptions{Password: password}) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + u, err := client.User(ctxChild, user) + cancel() + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if driver.IsNotFound(err) { + _, err := client.CreateUser(ctxChild, user, &driver.UserOptions{Password: password}) return password, errors.WithStack(err) } else if err == nil { - return password, errors.WithStack(u.Update(context.Background(), driver.UserOptions{ + return password, errors.WithStack(u.Update(ctxChild, driver.UserOptions{ Password: password, })) } else { @@ -122,7 +134,7 @@ func (a actionBootstrapSetPassword) setUserPassword(ctx context.Context, user, s } } -func (a actionBootstrapSetPassword) ensureUserPasswordSecret(user, secret string) (string, error) { +func (a actionBootstrapSetPassword) ensureUserPasswordSecret(ctx context.Context, user, secret string) (string, error) { cache := a.actionCtx.GetCachedStatus() if auth, ok := cache.Secret(secret); !ok { @@ -134,7 +146,8 @@ func (a actionBootstrapSetPassword) ensureUserPasswordSecret(user, secret string token := hex.EncodeToString(tokenData) owner := a.actionCtx.GetAPIObject().AsOwner() - if err := k8sutil.CreateBasicAuthSecret(a.actionCtx.SecretsInterface(), secret, user, token, &owner); err != nil { + err := k8sutil.CreateBasicAuthSecret(ctx, a.actionCtx.SecretsInterface(), secret, user, token, &owner) + if err != nil { return "", err } diff --git a/pkg/deployment/reconcile/action_bootstrap_update.go b/pkg/deployment/reconcile/action_bootstrap_update.go index 0b92a33e6..33d528c4d 100644 --- a/pkg/deployment/reconcile/action_bootstrap_update.go +++ b/pkg/deployment/reconcile/action_bootstrap_update.go @@ -51,7 +51,7 @@ type actionBootstrapUpdate struct { } func (a actionBootstrapUpdate) Start(ctx context.Context) (bool, error) { - if err := a.actionCtx.WithStatusUpdate(func(status *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(status *api.DeploymentStatus) bool { if errMessage, ok := a.action.GetParam("error"); ok { status.Conditions.Update(api.ConditionTypeBootstrapCompleted, true, "Bootstrap failed", fmt.Sprintf("%s", errMessage)) status.Conditions.Update(api.ConditionTypeBootstrapSucceded, false, "Bootstrap failed", fmt.Sprintf("%s", errMessage)) diff --git a/pkg/deployment/reconcile/action_cleanout_member.go b/pkg/deployment/reconcile/action_cleanout_member.go index 69f8c3af7..2f4417263 100644 --- a/pkg/deployment/reconcile/action_cleanout_member.go +++ b/pkg/deployment/reconcile/action_cleanout_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -64,19 +65,28 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { return true, nil } log := a.log - c, err := a.actionCtx.GetDatabaseClient(ctx) + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, errors.WithStack(err) } - cluster, err := c.Cluster(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cluster, err := c.Cluster(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") return false, errors.WithStack(err) } + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() var jobID string - ctx = driver.WithJobIDResponse(ctx, &jobID) - if err := cluster.CleanOutServer(ctx, a.action.MemberID); err != nil { + ctxJobID := driver.WithJobIDResponse(ctxChild, &jobID) + if err := cluster.CleanOutServer(ctxJobID, a.action.MemberID); err != nil { log.Debug().Err(err).Msg("Failed to cleanout member") return false, errors.WithStack(err) } @@ -84,7 +94,7 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseCleanOut m.CleanoutJobID = jobID - if a.actionCtx.UpdateMember(m); err != nil { + if a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil @@ -103,17 +113,26 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e if !m.IsInitialized { return true, false, nil } - c, err := a.actionCtx.GetDatabaseClient(ctx) + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create database client") return false, false, errors.WithStack(err) } - cluster, err := c.Cluster(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cluster, err := c.Cluster(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") return false, false, errors.WithStack(err) } - cleanedOut, err := cluster.IsCleanedOut(ctx, a.action.MemberID) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cleanedOut, err := cluster.IsCleanedOut(ctxChild, a.action.MemberID) + cancel() if err != nil { log.Debug().Err(err).Msg("IsCleanedOut failed") return false, false, errors.WithStack(err) @@ -122,17 +141,25 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e // We're not done yet, check job status log.Debug().Msg("IsCleanedOut returned false") - c, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create database client") return false, false, errors.WithStack(err) } - agency, err := a.actionCtx.GetAgency(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + agency, err := a.actionCtx.GetAgency(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return false, false, errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, m.CleanoutJobID, c, agency) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch cleanout job status") return false, false, errors.WithStack(err) @@ -142,7 +169,7 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e // Revert cleanout state m.Phase = api.MemberPhaseCreated m.CleanoutJobID = "" - if a.actionCtx.UpdateMember(m); err != nil { + if a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return false, true, nil @@ -151,7 +178,7 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e } // Cleanout completed if m.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") { - if a.actionCtx.UpdateMember(m); err != nil { + if a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } } diff --git a/pkg/deployment/reconcile/action_cluster_member_cleanup.go b/pkg/deployment/reconcile/action_cluster_member_cleanup.go index 411450a69..feb08d3e4 100644 --- a/pkg/deployment/reconcile/action_cluster_member_cleanup.go +++ b/pkg/deployment/reconcile/action_cluster_member_cleanup.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -68,17 +71,23 @@ func (a *actionClusterMemberCleanup) Start(ctx context.Context) (bool, error) { func (a *actionClusterMemberCleanup) start(ctx context.Context) error { id := driver.ServerID(a.MemberID()) - c, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { return err } - cluster, err := c.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cluster, err := c.Cluster(ctxChild) + cancel() if err != nil { return err } - health, err := cluster.Health(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + health, err := cluster.Health(ctxChild) + cancel() if err != nil { return err } @@ -87,7 +96,10 @@ func (a *actionClusterMemberCleanup) start(ctx context.Context) error { return nil } - if err := cluster.RemoveServer(ctx, id); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + + if err := cluster.RemoveServer(ctxChild, id); err != nil { return err } diff --git a/pkg/deployment/reconcile/action_context.go b/pkg/deployment/reconcile/action_context.go index 9364516c7..ce413becc 100644 --- a/pkg/deployment/reconcile/action_context.go +++ b/pkg/deployment/reconcile/action_context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -71,33 +72,33 @@ type ActionContext interface { GetMemberStatusByID(id string) (api.MemberStatus, bool) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. - CreateMember(group api.ServerGroup, id string) (string, error) + CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) // UpdateMember updates the deployment status wrt the given member. - UpdateMember(member api.MemberStatus) error + UpdateMember(ctx context.Context, member api.MemberStatus) error // RemoveMemberByID removes a member with given id. - RemoveMemberByID(id string) error + RemoveMemberByID(ctx context.Context, id string) error // GetPod returns pod. - GetPod(podName string) (*v1.Pod, error) + GetPod(ctx context.Context, podName string) (*v1.Pod, error) // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. - DeletePvc(pvcName string) error + DeletePvc(ctx context.Context, pvcName string) error // GetPvc returns PVC info about PVC with given name in the namespace // of the deployment. - GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) // UpdatePvc update PVC with given name in the namespace // of the deployment. - UpdatePvc(pvc *v1.PersistentVolumeClaim) error + UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - RemovePodFinalizers(podName string) error + RemovePodFinalizers(ctx context.Context, podName string) error // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. - DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error + DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error // DeleteTLSCASecret removes the Secret containing the TLS CA certificate. - DeleteTLSCASecret() error + DeleteTLSCASecret(ctx context.Context) error // GetImageInfo returns the image info for an image with given name. // Returns: (info, infoFound) GetImageInfo(imageName string) (api.ImageInfo, bool) @@ -106,7 +107,7 @@ type ActionContext interface { GetCurrentImageInfo() (api.ImageInfo, bool) // SetCurrentImage changes the CurrentImage field in the deployment // status to the given image. - SetCurrentImage(imageInfo api.ImageInfo) error + SetCurrentImage(ctx context.Context, imageInfo api.ImageInfo) error // GetDeploymentHealth returns a copy of the latest known state of cluster health GetDeploymentHealth() (driver.ClusterHealth, error) // GetShardSyncStatus returns true if all shards are in sync @@ -118,16 +119,16 @@ type ActionContext interface { // GetStatus returns a copy of the status GetStatus() api.DeploymentStatus // DisableScalingCluster disables scaling DBservers and coordinators - DisableScalingCluster() error + DisableScalingCluster(ctx context.Context) error // EnableScalingCluster enables scaling DBservers and coordinators - EnableScalingCluster() error + EnableScalingCluster(ctx context.Context) error // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error + UpdateClusterCondition(ctx context.Context, conditionType api.ConditionType, status bool, reason, message string) error SecretsInterface() k8sutil.SecretInterface // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives information about a deployment name GetName() string // GetNameget current cached state of deployment @@ -166,12 +167,12 @@ func (ac *actionContext) GetStatus() api.DeploymentStatus { return *s } -func (ac *actionContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) { - return ac.context.GetBackup(backup) +func (ac *actionContext) GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) { + return ac.context.GetBackup(ctx, backup) } -func (ac *actionContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { - return ac.context.WithStatusUpdate(action, force...) +func (ac *actionContext) WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error { + return ac.context.WithStatusUpdate(ctx, action, force...) } func (ac *actionContext) SecretsInterface() k8sutil.SecretInterface { @@ -182,8 +183,8 @@ func (ac *actionContext) GetShardSyncStatus() bool { return ac.context.GetShardSyncStatus() } -func (ac *actionContext) UpdateClusterCondition(conditionType api.ConditionType, status bool, reason, message string) error { - return ac.context.WithStatusUpdate(func(s *api.DeploymentStatus) bool { +func (ac *actionContext) UpdateClusterCondition(ctx context.Context, conditionType api.ConditionType, status bool, reason, message string) error { + return ac.context.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { return s.Conditions.Update(conditionType, status, reason, message) }) } @@ -192,16 +193,16 @@ func (ac *actionContext) GetAPIObject() k8sutil.APIObject { return ac.context.GetAPIObject() } -func (ac *actionContext) UpdatePvc(pvc *v1.PersistentVolumeClaim) error { - return ac.context.UpdatePvc(pvc) +func (ac *actionContext) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { + return ac.context.UpdatePvc(ctx, pvc) } func (ac *actionContext) CreateEvent(evt *k8sutil.Event) { ac.context.CreateEvent(evt) } -func (ac *actionContext) GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) { - return ac.context.GetPvc(pvcName) +func (ac *actionContext) GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) { + return ac.context.GetPvc(ctx, pvcName) } // Gets the specified mode of deployment @@ -276,8 +277,8 @@ func (ac *actionContext) GetMemberStatusByID(id string) (api.MemberStatus, bool) // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. -func (ac *actionContext) CreateMember(group api.ServerGroup, id string) (string, error) { - result, err := ac.context.CreateMember(group, id) +func (ac *actionContext) CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) { + result, err := ac.context.CreateMember(ctx, group, id) if err != nil { return "", errors.WithStack(err) } @@ -285,7 +286,7 @@ func (ac *actionContext) CreateMember(group api.ServerGroup, id string) (string, } // UpdateMember updates the deployment status wrt the given member. -func (ac *actionContext) UpdateMember(member api.MemberStatus) error { +func (ac *actionContext) UpdateMember(ctx context.Context, member api.MemberStatus) error { status, lastVersion := ac.context.GetStatus() _, group, found := status.Members.ElementByID(member.ID) if !found { @@ -294,7 +295,7 @@ func (ac *actionContext) UpdateMember(member api.MemberStatus) error { if err := status.Members.Update(member, group); err != nil { return errors.WithStack(err) } - if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + if err := ac.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Updating CR status failed") return errors.WithStack(err) } @@ -302,7 +303,7 @@ func (ac *actionContext) UpdateMember(member api.MemberStatus) error { } // RemoveMemberByID removes a member with given id. -func (ac *actionContext) RemoveMemberByID(id string) error { +func (ac *actionContext) RemoveMemberByID(ctx context.Context, id string) error { status, lastVersion := ac.context.GetStatus() _, group, found := status.Members.ElementByID(id) if !found { @@ -313,15 +314,15 @@ func (ac *actionContext) RemoveMemberByID(id string) error { return errors.WithStack(err) } // Save removed member - if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + if err := ac.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } return nil } // GetPod returns pod. -func (ac *actionContext) GetPod(podName string) (*v1.Pod, error) { - if pod, err := ac.context.GetPod(podName); err != nil { +func (ac *actionContext) GetPod(ctx context.Context, podName string) (*v1.Pod, error) { + if pod, err := ac.context.GetPod(ctx, podName); err != nil { return nil, errors.WithStack(err) } else { return pod, nil @@ -330,8 +331,8 @@ func (ac *actionContext) GetPod(podName string) (*v1.Pod, error) { // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (ac *actionContext) DeletePod(podName string) error { - if err := ac.context.DeletePod(podName); err != nil { +func (ac *actionContext) DeletePod(ctx context.Context, podName string) error { + if err := ac.context.DeletePod(ctx, podName); err != nil { return errors.WithStack(err) } return nil @@ -339,8 +340,8 @@ func (ac *actionContext) DeletePod(podName string) error { // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. -func (ac *actionContext) DeletePvc(pvcName string) error { - if err := ac.context.DeletePvc(pvcName); err != nil { +func (ac *actionContext) DeletePvc(ctx context.Context, pvcName string) error { + if err := ac.context.DeletePvc(ctx, pvcName); err != nil { return errors.WithStack(err) } return nil @@ -348,8 +349,8 @@ func (ac *actionContext) DeletePvc(pvcName string) error { // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. -func (ac *actionContext) RemovePodFinalizers(podName string) error { - if err := ac.context.RemovePodFinalizers(podName); err != nil { +func (ac *actionContext) RemovePodFinalizers(ctx context.Context, podName string) error { + if err := ac.context.RemovePodFinalizers(ctx, podName); err != nil { return errors.WithStack(err) } return nil @@ -357,15 +358,15 @@ func (ac *actionContext) RemovePodFinalizers(podName string) error { // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. -func (ac *actionContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { - if err := ac.context.DeleteTLSKeyfile(group, member); err != nil { +func (ac *actionContext) DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error { + if err := ac.context.DeleteTLSKeyfile(ctx, group, member); err != nil { return errors.WithStack(err) } return nil } // DeleteTLSCASecret removes the Secret containing the TLS CA certificate. -func (ac *actionContext) DeleteTLSCASecret() error { +func (ac *actionContext) DeleteTLSCASecret(ctx context.Context) error { spec := ac.context.GetSpec().TLS if !spec.IsSecure() { return nil @@ -378,7 +379,7 @@ func (ac *actionContext) DeleteTLSCASecret() error { status, lastVersion := ac.context.GetStatus() if status.SecretHashes != nil { status.SecretHashes.TLSCA = "" - if err := ac.context.UpdateStatus(status, lastVersion); err != nil { + if err := ac.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -410,8 +411,8 @@ func (ac *actionContext) GetCurrentImageInfo() (api.ImageInfo, bool) { // SetCurrentImage changes the CurrentImage field in the deployment // status to the given image. -func (ac *actionContext) SetCurrentImage(imageInfo api.ImageInfo) error { - return ac.context.WithStatusUpdate(func(s *api.DeploymentStatus) bool { +func (ac *actionContext) SetCurrentImage(ctx context.Context, imageInfo api.ImageInfo) error { + return ac.context.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.CurrentImage == nil || s.CurrentImage.Image != imageInfo.Image { s.CurrentImage = &imageInfo return true @@ -426,11 +427,11 @@ func (ac *actionContext) InvalidateSyncStatus() { } // DisableScalingCluster disables scaling DBservers and coordinators -func (ac *actionContext) DisableScalingCluster() error { - return ac.context.DisableScalingCluster() +func (ac *actionContext) DisableScalingCluster(ctx context.Context) error { + return ac.context.DisableScalingCluster(ctx) } // EnableScalingCluster enables scaling DBservers and coordinators -func (ac *actionContext) EnableScalingCluster() error { - return ac.context.EnableScalingCluster() +func (ac *actionContext) EnableScalingCluster(ctx context.Context) error { + return ac.context.EnableScalingCluster(ctx) } diff --git a/pkg/deployment/reconcile/action_disable_scaling_cluster.go b/pkg/deployment/reconcile/action_disable_scaling_cluster.go index d2c8094a6..e7493cf70 100644 --- a/pkg/deployment/reconcile/action_disable_scaling_cluster.go +++ b/pkg/deployment/reconcile/action_disable_scaling_cluster.go @@ -54,7 +54,7 @@ type actionDisableScalingCluster struct { // Start disables scaling DBservers and coordinators func (a *actionDisableScalingCluster) Start(ctx context.Context) (bool, error) { - err := a.actionCtx.DisableScalingCluster() + err := a.actionCtx.DisableScalingCluster(ctx) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_enable_scaling_cluster.go b/pkg/deployment/reconcile/action_enable_scaling_cluster.go index 5f05ca763..178c4357d 100644 --- a/pkg/deployment/reconcile/action_enable_scaling_cluster.go +++ b/pkg/deployment/reconcile/action_enable_scaling_cluster.go @@ -54,7 +54,7 @@ type actionEnableScalingCluster struct { // Start enables scaling DBservers and coordinators func (a *actionEnableScalingCluster) Start(ctx context.Context) (bool, error) { - err := a.actionCtx.EnableScalingCluster() + err := a.actionCtx.EnableScalingCluster(ctx) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_encryption_add.go b/pkg/deployment/reconcile/action_encryption_add.go index c8df1c277..62a83f479 100644 --- a/pkg/deployment/reconcile/action_encryption_add.go +++ b/pkg/deployment/reconcile/action_encryption_add.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,8 @@ import ( "context" "encoding/base64" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -85,7 +88,7 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) { secret = s } - sha, d, exists, err := pod.GetEncryptionKey(a.actionCtx.SecretsInterface(), secret) + sha, d, exists, err := pod.GetEncryptionKey(ctx, a.actionCtx.SecretsInterface(), secret) if err != nil { a.log.Error().Err(err).Msgf("Unable to fetch current encryption key") return true, nil @@ -104,7 +107,10 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_encryption_propagated.go b/pkg/deployment/reconcile/action_encryption_propagated.go index 45aad362a..8a41c815a 100644 --- a/pkg/deployment/reconcile/action_encryption_propagated.go +++ b/pkg/deployment/reconcile/action_encryption_propagated.go @@ -56,7 +56,7 @@ func (a *encryptionKeyPropagatedAction) Start(ctx context.Context) (bool, error) propagatedFlagBool := propagatedFlag == conditionTrue - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.Encryption.Propagated != propagatedFlagBool { s.Hashes.Encryption.Propagated = propagatedFlagBool return true diff --git a/pkg/deployment/reconcile/action_encryption_refresh.go b/pkg/deployment/reconcile/action_encryption_refresh.go index d829cf795..acb44df4a 100644 --- a/pkg/deployment/reconcile/action_encryption_refresh.go +++ b/pkg/deployment/reconcile/action_encryption_refresh.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,11 +26,14 @@ package reconcile import ( "context" + "github.com/rs/zerolog" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/client" "github.com/arangodb/kube-arangodb/pkg/deployment/pod" - "github.com/rs/zerolog" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) func init() { @@ -54,21 +58,26 @@ func (a *encryptionKeyRefreshAction) Start(ctx context.Context) (bool, error) { } func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) { - keyfolder, err := a.actionCtx.SecretsInterface().Get(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + keyfolder, err := a.actionCtx.SecretsInterface().Get(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) + cancel() if err != nil { a.log.Err(err).Msgf("Unable to fetch encryption folder") return true, false, nil } - c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) + cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil } client := client.NewClient(c.Connection()) - - e, err := client.RefreshEncryption(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + e, err := client.RefreshEncryption(ctxChild) + cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to refresh encryption") return true, false, nil diff --git a/pkg/deployment/reconcile/action_encryption_remove.go b/pkg/deployment/reconcile/action_encryption_remove.go index e8eb0f1f6..9b7d44af5 100644 --- a/pkg/deployment/reconcile/action_encryption_remove.go +++ b/pkg/deployment/reconcile/action_encryption_remove.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -81,7 +82,10 @@ func (a *encryptionKeyRemoveAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_encryption_status_update.go b/pkg/deployment/reconcile/action_encryption_status_update.go index 4782a1f4e..63ecdbf08 100644 --- a/pkg/deployment/reconcile/action_encryption_status_update.go +++ b/pkg/deployment/reconcile/action_encryption_status_update.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -57,7 +60,10 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro return true, nil } - f, err := a.actionCtx.SecretsInterface().Get(ctx, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + f, err := a.actionCtx.SecretsInterface().Get(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), meta.GetOptions{}) if err != nil { a.log.Error().Err(err).Msgf("Unable to get folder info") return true, nil @@ -65,7 +71,7 @@ func (a *encryptionKeyStatusUpdateAction) Start(ctx context.Context) (bool, erro keyHashes := secretKeysToListWithPrefix("sha256:", f) - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if len(keyHashes) == 0 { if s.Hashes.Encryption.Keys != nil { s.Hashes.Encryption.Keys = nil diff --git a/pkg/deployment/reconcile/action_helper.go b/pkg/deployment/reconcile/action_helper.go index 5a4441980..8ce21d571 100644 --- a/pkg/deployment/reconcile/action_helper.go +++ b/pkg/deployment/reconcile/action_helper.go @@ -43,14 +43,14 @@ type actionEmptyCheckProgress struct { // CheckProgress define optional check progress for action // Returns: ready, abort, error. -func (e actionEmptyCheckProgress) CheckProgress(ctx context.Context) (bool, bool, error) { +func (e actionEmptyCheckProgress) CheckProgress(_ context.Context) (bool, bool, error) { return true, false, nil } type actionEmptyStart struct { } -func (e actionEmptyStart) Start(ctx context.Context) (bool, error) { +func (e actionEmptyStart) Start(_ context.Context) (bool, error) { return false, nil } diff --git a/pkg/deployment/reconcile/action_idle.go b/pkg/deployment/reconcile/action_idle.go index 9fa513841..c54d7ab08 100644 --- a/pkg/deployment/reconcile/action_idle.go +++ b/pkg/deployment/reconcile/action_idle.go @@ -55,6 +55,6 @@ type actionIdle struct { // Start performs the start of the action. // Returns true if the action is completely finished, false in case // the start time needs to be recorded and a ready condition needs to be checked. -func (a *actionIdle) Start(ctx context.Context) (bool, error) { +func (a *actionIdle) Start(_ context.Context) (bool, error) { return true, nil } diff --git a/pkg/deployment/reconcile/action_jwt_add.go b/pkg/deployment/reconcile/action_jwt_add.go index 5ecfe52d9..dc2fd210c 100644 --- a/pkg/deployment/reconcile/action_jwt_add.go +++ b/pkg/deployment/reconcile/action_jwt_add.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -116,7 +117,10 @@ func (a *jwtAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_clean.go b/pkg/deployment/reconcile/action_jwt_clean.go index 169c9d58e..b27b143f6 100644 --- a/pkg/deployment/reconcile/action_jwt_clean.go +++ b/pkg/deployment/reconcile/action_jwt_clean.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -107,7 +108,10 @@ func (a *jwtCleanAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_propagated.go b/pkg/deployment/reconcile/action_jwt_propagated.go index ffc56aa34..a39e1bdb1 100644 --- a/pkg/deployment/reconcile/action_jwt_propagated.go +++ b/pkg/deployment/reconcile/action_jwt_propagated.go @@ -62,7 +62,7 @@ func (a *jwtPropagatedAction) Start(ctx context.Context) (bool, error) { propagatedFlagBool := propagatedFlag == conditionTrue - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.JWT.Propagated != propagatedFlagBool { s.Hashes.JWT.Propagated = propagatedFlagBool return true diff --git a/pkg/deployment/reconcile/action_jwt_refresh.go b/pkg/deployment/reconcile/action_jwt_refresh.go index d69533273..71b2dcdca 100644 --- a/pkg/deployment/reconcile/action_jwt_refresh.go +++ b/pkg/deployment/reconcile/action_jwt_refresh.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -28,6 +29,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/client" "github.com/arangodb/kube-arangodb/pkg/deployment/pod" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/rs/zerolog" ) @@ -58,12 +60,17 @@ func (a *jwtRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error return true, false, nil } - c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) + cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil } - if invalid, err := isMemberJWTTokenInvalid(ctx, client.NewClient(c.Connection()), folder.Data, true); err != nil { + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if invalid, err := isMemberJWTTokenInvalid(ctxChild, client.NewClient(c.Connection()), folder.Data, true); err != nil { a.log.Warn().Err(err).Msg("Error while getting JWT Status") return true, false, nil } else if invalid { diff --git a/pkg/deployment/reconcile/action_jwt_set_active.go b/pkg/deployment/reconcile/action_jwt_set_active.go index 17245d98c..ae7a88fd4 100644 --- a/pkg/deployment/reconcile/action_jwt_set_active.go +++ b/pkg/deployment/reconcile/action_jwt_set_active.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -118,7 +119,10 @@ func (a *jwtSetActiveAction) Start(ctx context.Context) (bool, error) { return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_status_update.go b/pkg/deployment/reconcile/action_jwt_status_update.go index 7cf948c0b..79e4793ea 100644 --- a/pkg/deployment/reconcile/action_jwt_status_update.go +++ b/pkg/deployment/reconcile/action_jwt_status_update.go @@ -106,7 +106,7 @@ func (a *jwtStatusUpdateAction) Start(ctx context.Context) (bool, error) { keySha := fmt.Sprintf("sha256:%s", util.SHA256(key)) - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.JWT.Passive != nil { s.Hashes.JWT.Passive = nil return true @@ -131,7 +131,7 @@ func (a *jwtStatusUpdateAction) Start(ctx context.Context) (bool, error) { return true, nil } - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) (update bool) { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) (update bool) { activeKeyData, active := f.Data[pod.ActiveJWTKey] activeKeyShort := util.SHA256(activeKeyData) activeKey := fmt.Sprintf("sha256:%s", activeKeyShort) diff --git a/pkg/deployment/reconcile/action_maintenance_disable.go b/pkg/deployment/reconcile/action_maintenance_disable.go index 4e7e53ce9..2927a9de0 100644 --- a/pkg/deployment/reconcile/action_maintenance_disable.go +++ b/pkg/deployment/reconcile/action_maintenance_disable.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,7 @@ import ( api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/rs/zerolog" ) @@ -57,13 +59,17 @@ func (a *actionDisableMaintenance) Start(ctx context.Context) (bool, error) { return true, nil } - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { a.log.Error().Err(err).Msgf("Unable to get agency client") return true, nil } - if err := agency.SetMaintenanceMode(ctx, client, false); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if err := agency.SetMaintenanceMode(ctxChild, client, false); err != nil { a.log.Error().Err(err).Msgf("Unable to disable maintenance") return true, nil } diff --git a/pkg/deployment/reconcile/action_maintenance_enable.go b/pkg/deployment/reconcile/action_maintenance_enable.go index ceb3d1b68..d032f63e0 100644 --- a/pkg/deployment/reconcile/action_maintenance_enable.go +++ b/pkg/deployment/reconcile/action_maintenance_enable.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" "github.com/rs/zerolog" @@ -57,13 +60,17 @@ func (a *actionEnableMaintenance) Start(ctx context.Context) (bool, error) { return true, nil } - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { a.log.Error().Err(err).Msgf("Unable to get agency client") return true, nil } - if err := agency.SetMaintenanceMode(ctx, client, true); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if err := agency.SetMaintenanceMode(ctxChild, client, true); err != nil { a.log.Error().Err(err).Msgf("Unable to set maintenance") return true, nil } diff --git a/pkg/deployment/reconcile/action_mark_to_remove_member.go b/pkg/deployment/reconcile/action_mark_to_remove_member.go index f02a6808a..8bea24d1a 100644 --- a/pkg/deployment/reconcile/action_mark_to_remove_member.go +++ b/pkg/deployment/reconcile/action_mark_to_remove_member.go @@ -54,7 +54,7 @@ func (a *actionMarkToRemove) Start(ctx context.Context) (bool, error) { return true, nil } - return true, a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + return true, a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { member, group, ok := s.Members.ElementByID(a.action.MemberID) if !ok { return false diff --git a/pkg/deployment/reconcile/action_pvc_resize.go b/pkg/deployment/reconcile/action_pvc_resize.go index 4d547bfd6..89a34b868 100644 --- a/pkg/deployment/reconcile/action_pvc_resize.go +++ b/pkg/deployment/reconcile/action_pvc_resize.go @@ -71,7 +71,7 @@ func (a *actionPVCResize) Start(ctx context.Context) (bool, error) { return true, nil } - pvc, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + pvc, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if apiErrors.IsNotFound(err) { return true, nil @@ -92,14 +92,14 @@ func (a *actionPVCResize) Start(ctx context.Context) (bool, error) { cmp := volumeSize.Cmp(requestedSize) if cmp < 0 { pvc.Spec.Resources.Requests[core.ResourceStorage] = requestedSize - if err := a.actionCtx.UpdatePvc(pvc); err != nil { + if err := a.actionCtx.UpdatePvc(ctx, pvc); err != nil { return false, err } return false, nil } else if cmp > 0 { if groupSpec.GetVolumeAllowShrink() && group == api.ServerGroupDBServers { - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { s.Plan = append(s.Plan, api.NewAction(api.ActionTypeMarkToRemoveMember, group, m.ID)) return true }); err != nil { @@ -129,7 +129,7 @@ func (a *actionPVCResize) CheckProgress(ctx context.Context) (bool, bool, error) return true, false, nil } - pvc, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + pvc, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if apiErrors.IsNotFound(err) { return true, false, nil diff --git a/pkg/deployment/reconcile/action_pvc_resized.go b/pkg/deployment/reconcile/action_pvc_resized.go index 00a51ae2f..db901b375 100644 --- a/pkg/deployment/reconcile/action_pvc_resized.go +++ b/pkg/deployment/reconcile/action_pvc_resized.go @@ -67,7 +67,7 @@ func (a *actionPVCResized) CheckProgress(ctx context.Context) (bool, bool, error return true, false, nil } - pvc, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + pvc, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if apiErrors.IsNotFound(err) { return true, false, nil diff --git a/pkg/deployment/reconcile/action_recreate_member.go b/pkg/deployment/reconcile/action_recreate_member.go index 18deb38d3..a3268ff48 100644 --- a/pkg/deployment/reconcile/action_recreate_member.go +++ b/pkg/deployment/reconcile/action_recreate_member.go @@ -66,7 +66,7 @@ func (a *actionRecreateMember) Start(ctx context.Context) (bool, error) { return false, errors.Newf("expecting member to be present in list, but it is not") } - _, err := a.actionCtx.GetPvc(m.PersistentVolumeClaimName) + _, err := a.actionCtx.GetPvc(ctx, m.PersistentVolumeClaimName) if err != nil { if kubeErrors.IsNotFound(err) { return false, errors.Newf("PVC is missing %s. Members won't be recreated without old PV", m.PersistentVolumeClaimName) @@ -80,7 +80,7 @@ func (a *actionRecreateMember) Start(ctx context.Context) (bool, error) { m.Phase = api.MemberPhaseNone } - if err = a.actionCtx.UpdateMember(m); err != nil { + if err = a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } diff --git a/pkg/deployment/reconcile/action_remove_member.go b/pkg/deployment/reconcile/action_remove_member.go index 3734e3ce2..28e2958a6 100644 --- a/pkg/deployment/reconcile/action_remove_member.go +++ b/pkg/deployment/reconcile/action_remove_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -68,11 +69,16 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) { } // For safety, remove from cluster if a.action.Group == api.ServerGroupCoordinators || a.action.Group == api.ServerGroupDBServers { - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { return false, errors.WithStack(err) } - if err := arangod.RemoveServerFromCluster(ctx, client.Connection(), driver.ServerID(m.ID)); err != nil { + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if err := arangod.RemoveServerFromCluster(ctxChild, client.Connection(), driver.ServerID(m.ID)); err != nil { if !driver.IsNotFound(err) && !driver.IsPreconditionFailed(err) { a.log.Err(err).Str("member-id", m.ID).Msgf("Failed to remove server from cluster") // ignore this error, maybe all coordinators are failed and no connction to cluster is possible @@ -100,17 +106,17 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) { } } // Remove the pod (if any) - if err := a.actionCtx.DeletePod(m.PodName); err != nil { + if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil { return false, errors.WithStack(err) } // Remove the pvc (if any) if m.PersistentVolumeClaimName != "" { - if err := a.actionCtx.DeletePvc(m.PersistentVolumeClaimName); err != nil { + if err := a.actionCtx.DeletePvc(ctx, m.PersistentVolumeClaimName); err != nil { return false, errors.WithStack(err) } } // Remove member - if err := a.actionCtx.RemoveMemberByID(a.action.MemberID); err != nil { + if err := a.actionCtx.RemoveMemberByID(ctx, a.action.MemberID); err != nil { return false, errors.WithStack(err) } // Check that member has been removed diff --git a/pkg/deployment/reconcile/action_resign_leadership.go b/pkg/deployment/reconcile/action_resign_leadership.go index 2f1744f89..bef0fd301 100644 --- a/pkg/deployment/reconcile/action_resign_leadership.go +++ b/pkg/deployment/reconcile/action_resign_leadership.go @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -68,7 +69,9 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { return true, nil } - client, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get client") return true, errors.WithStack(err) @@ -76,14 +79,18 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { switch group { case api.ServerGroupDBServers: - cluster, err := client.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cluster, err := client.Cluster(ctxChild) + cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get cluster client") return true, errors.WithStack(err) } var jobID string - jobCtx := driver.WithJobIDResponse(ctx, &jobID) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + jobCtx := driver.WithJobIDResponse(ctxChild, &jobID) log.Debug().Msg("Temporary shutdown, resign leadership") if err := cluster.ResignServer(jobCtx, m.ID); err != nil { log.Debug().Err(err).Msg("Failed to resign server") @@ -92,7 +99,7 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { m.CleanoutJobID = jobID - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return true, errors.WithStack(err) } @@ -112,19 +119,25 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, return true, false, nil } - agency, err := a.actionCtx.GetAgency(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + agency, err := a.actionCtx.GetAgency(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return false, false, errors.WithStack(err) } - c, err := a.actionCtx.GetDatabaseClient(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, false, errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, m.CleanoutJobID, c, agency) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency) + cancel() if err != nil { if driver.IsNotFound(err) { log.Debug().Err(err).Msg("Job not found, but proceeding") @@ -136,7 +149,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, if jobStatus.IsFailed() { m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } log.Error().Msg("Resign server job failed") @@ -145,7 +158,7 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, if jobStatus.IsFinished() { m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return true, false, nil diff --git a/pkg/deployment/reconcile/action_rotate_member.go b/pkg/deployment/reconcile/action_rotate_member.go index c9f51f3bc..6ed715ad7 100644 --- a/pkg/deployment/reconcile/action_rotate_member.go +++ b/pkg/deployment/reconcile/action_rotate_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -72,7 +73,7 @@ func (a *actionRotateMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseRotating - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil @@ -96,7 +97,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err } // Pod is terminated, we can now remove it - if err := a.actionCtx.DeletePod(m.PodName); err != nil { + if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil { if !k8sutil.IsNotFound(err) { return false, false, errors.WithStack(err) } @@ -105,7 +106,7 @@ func (a *actionRotateMember) CheckProgress(ctx context.Context) (bool, bool, err m.Phase = api.MemberPhaseNone m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return true, false, nil diff --git a/pkg/deployment/reconcile/action_rotate_start_member.go b/pkg/deployment/reconcile/action_rotate_start_member.go index 73e0188db..259d201db 100644 --- a/pkg/deployment/reconcile/action_rotate_start_member.go +++ b/pkg/deployment/reconcile/action_rotate_start_member.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -72,7 +73,7 @@ func (a *actionRotateStartMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseRotateStart - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil @@ -96,7 +97,7 @@ func (a *actionRotateStartMember) CheckProgress(ctx context.Context) (bool, bool } // Pod is terminated, we can now remove it - if err := a.actionCtx.DeletePod(m.PodName); err != nil { + if err := a.actionCtx.DeletePod(ctx, m.PodName); err != nil { if !k8sutil.IsNotFound(err) { return false, false, errors.WithStack(err) } diff --git a/pkg/deployment/reconcile/action_rotate_stop_member.go b/pkg/deployment/reconcile/action_rotate_stop_member.go index 3cc86c331..a155e1047 100644 --- a/pkg/deployment/reconcile/action_rotate_stop_member.go +++ b/pkg/deployment/reconcile/action_rotate_stop_member.go @@ -67,7 +67,7 @@ func (a *actionRotateStopMember) Start(ctx context.Context) (bool, error) { m.Phase = api.MemberPhaseNone m.RecentTerminations = nil // Since we're rotating, we do not care about old terminations. m.CleanoutJobID = "" - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil diff --git a/pkg/deployment/reconcile/action_set_current_image.go b/pkg/deployment/reconcile/action_set_current_image.go index 816612d1c..aed10d7f0 100644 --- a/pkg/deployment/reconcile/action_set_current_image.go +++ b/pkg/deployment/reconcile/action_set_current_image.go @@ -73,7 +73,7 @@ func (a *setCurrentMemberImageAction) CheckProgress(ctx context.Context) (bool, return true, false, nil } - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { m, g, found := s.Members.ElementByID(a.action.MemberID) if !found { log.Error().Msg("No such member") diff --git a/pkg/deployment/reconcile/action_shutdown_member.go b/pkg/deployment/reconcile/action_shutdown_member.go index 06ff2dde6..f3cf49321 100644 --- a/pkg/deployment/reconcile/action_shutdown_member.go +++ b/pkg/deployment/reconcile/action_shutdown_member.go @@ -70,7 +70,7 @@ func (a *actionShutdownMember) Start(ctx context.Context) (bool, error) { // Update status m.Phase = api.MemberPhaseShuttingDown - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } return false, nil diff --git a/pkg/deployment/reconcile/action_tls_ca_append.go b/pkg/deployment/reconcile/action_tls_ca_append.go index 7eab3151f..2724a212d 100644 --- a/pkg/deployment/reconcile/action_tls_ca_append.go +++ b/pkg/deployment/reconcile/action_tls_ca_append.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -115,7 +116,10 @@ func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error) return true, nil } - _, err = a.actionCtx.SecretsInterface().Patch(ctx, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_tls_ca_clean.go b/pkg/deployment/reconcile/action_tls_ca_clean.go index 5615bdd97..a587420c9 100644 --- a/pkg/deployment/reconcile/action_tls_ca_clean.go +++ b/pkg/deployment/reconcile/action_tls_ca_clean.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -117,7 +118,11 @@ func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) { } a.log.Info().Msgf("Removing key %s from truststore", certChecksum) - _, err = a.actionCtx.SecretsInterface().Patch(ctx, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_tls_ca_renew.go b/pkg/deployment/reconcile/action_tls_ca_renew.go index 772a40c64..8cb646ddf 100644 --- a/pkg/deployment/reconcile/action_tls_ca_renew.go +++ b/pkg/deployment/reconcile/action_tls_ca_renew.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -54,8 +55,11 @@ func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) { return true, nil } + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + s := a.actionCtx.SecretsInterface() - if err := s.Delete(ctx, a.actionCtx.GetSpec().TLS.GetCASecretName(), meta.DeleteOptions{}); err != nil { + if err := s.Delete(ctxChild, a.actionCtx.GetSpec().TLS.GetCASecretName(), meta.DeleteOptions{}); err != nil { if !k8sutil.IsNotFound(err) { a.log.Warn().Err(err).Msgf("Unable to clean cert %s", a.actionCtx.GetSpec().TLS.GetCASecretName()) return true, nil diff --git a/pkg/deployment/reconcile/action_tls_keyfile_clean.go b/pkg/deployment/reconcile/action_tls_keyfile_clean.go index 54f174164..00f4da853 100644 --- a/pkg/deployment/reconcile/action_tls_keyfile_clean.go +++ b/pkg/deployment/reconcile/action_tls_keyfile_clean.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -60,7 +61,7 @@ func (a *cleanTLSKeyfileCertificateAction) Start(ctx context.Context) (bool, err return true, nil } - if err := a.actionCtx.DeleteTLSKeyfile(a.action.Group, member); err != nil { + if err := a.actionCtx.DeleteTLSKeyfile(ctx, a.action.Group, member); err != nil { a.log.Warn().Err(err).Msgf("Unable to remove keyfile") if !k8sutil.IsNotFound(err) { return false, err diff --git a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go index a8c0f38af..96cea6f3d 100644 --- a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go +++ b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/client" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" @@ -52,7 +54,9 @@ type refreshTLSKeyfileCertificateAction struct { } func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { - c, err := a.actionCtx.GetServerClient(ctx, a.action.Group, a.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) + cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil @@ -74,7 +78,9 @@ func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) client := client.NewClient(c.Connection()) - e, err := client.RefreshTLS(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + e, err := client.RefreshTLS(ctxChild) + cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to refresh TLS") return true, false, nil diff --git a/pkg/deployment/reconcile/action_tls_propagated.go b/pkg/deployment/reconcile/action_tls_propagated.go index d2f2fa7a4..0add6b289 100644 --- a/pkg/deployment/reconcile/action_tls_propagated.go +++ b/pkg/deployment/reconcile/action_tls_propagated.go @@ -56,7 +56,7 @@ func (a *tlsPropagatedAction) Start(ctx context.Context) (bool, error) { propagatedFlagBool := propagatedFlag == conditionTrue - if err := a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err := a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { if s.Hashes.TLS.Propagated != propagatedFlagBool { s.Hashes.TLS.Propagated = propagatedFlagBool return true diff --git a/pkg/deployment/reconcile/action_tls_sni_update.go b/pkg/deployment/reconcile/action_tls_sni_update.go index 8b61b3e0f..d930cbaf4 100644 --- a/pkg/deployment/reconcile/action_tls_sni_update.go +++ b/pkg/deployment/reconcile/action_tls_sni_update.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/rs/zerolog" ) @@ -68,13 +70,17 @@ func (t *tlsSNIUpdate) CheckProgress(ctx context.Context) (bool, bool, error) { return true, false, nil } - c, err := t.actionCtx.GetServerClient(ctx, t.action.Group, t.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := t.actionCtx.GetServerClient(ctxChild, t.action.Group, t.action.MemberID) + cancel() if err != nil { t.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil } - if ok, err := compareTLSSNIConfig(ctx, c.Connection(), fetchedSecrets, true); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if ok, err := compareTLSSNIConfig(ctxChild, c.Connection(), fetchedSecrets, true); err != nil { t.log.Warn().Err(err).Msg("Unable to compare TLS config") return true, false, nil } else { diff --git a/pkg/deployment/reconcile/action_tls_status_update.go b/pkg/deployment/reconcile/action_tls_status_update.go index c5a604d1b..4d70135e2 100644 --- a/pkg/deployment/reconcile/action_tls_status_update.go +++ b/pkg/deployment/reconcile/action_tls_status_update.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -27,6 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/deployment/resources" "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" meta "k8s.io/apimachinery/pkg/apis/meta/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" @@ -56,7 +58,9 @@ func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) { return true, nil } - f, err := a.actionCtx.SecretsInterface().Get(ctx, resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + f, err := a.actionCtx.SecretsInterface().Get(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{}) + cancel() if err != nil { a.log.Error().Err(err).Msgf("Unable to get folder info") return true, nil @@ -64,7 +68,7 @@ func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) { keyHashes := secretKeysToListWithPrefix("sha256:", f) - if err = a.actionCtx.WithStatusUpdate(func(s *api.DeploymentStatus) bool { + if err = a.actionCtx.WithStatusUpdate(ctx, func(s *api.DeploymentStatus) bool { r := false if len(keyHashes) == 1 { if s.Hashes.TLS.CA == nil || *s.Hashes.TLS.CA != keyHashes[0] { diff --git a/pkg/deployment/reconcile/action_upgrade_current_image.go b/pkg/deployment/reconcile/action_upgrade_current_image.go index 8cfa138ee..bfcb0a5c8 100644 --- a/pkg/deployment/reconcile/action_upgrade_current_image.go +++ b/pkg/deployment/reconcile/action_upgrade_current_image.go @@ -71,7 +71,7 @@ func (a *setCurrentImageAction) CheckProgress(ctx context.Context) (bool, bool, if !found { return false, false, nil } - if err := a.actionCtx.SetCurrentImage(imageInfo); err != nil { + if err := a.actionCtx.SetCurrentImage(ctx, imageInfo); err != nil { return false, false, errors.WithStack(err) } log.Info().Str("image", a.action.Image).Str("to", imageInfo.Image).Msg("Changed current main image") diff --git a/pkg/deployment/reconcile/action_upgrade_member.go b/pkg/deployment/reconcile/action_upgrade_member.go index d70807255..5a0cd679e 100644 --- a/pkg/deployment/reconcile/action_upgrade_member.go +++ b/pkg/deployment/reconcile/action_upgrade_member.go @@ -62,7 +62,7 @@ func (a *actionUpgradeMember) Start(ctx context.Context) (bool, error) { } // Set AutoUpgrade condition m.Conditions.Update(api.ConditionTypeAutoUpgrade, true, "Upgrading", "AutoUpgrade on first restart") - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, errors.WithStack(err) } @@ -112,7 +112,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, bool, er m.Image = m.OldImage.DeepCopy() } - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, true, nil } @@ -141,7 +141,7 @@ func (a *actionUpgradeMember) CheckProgress(ctx context.Context) (bool, bool, er if !m.OldImage.Equal(m.Image) && isUpgrading { m.OldImage = m.Image.DeepCopy() } - if err := a.actionCtx.UpdateMember(m); err != nil { + if err := a.actionCtx.UpdateMember(ctx, m); err != nil { return false, false, errors.WithStack(err) } return isUpgrading, false, nil diff --git a/pkg/deployment/reconcile/action_wait_for_member_up.go b/pkg/deployment/reconcile/action_wait_for_member_up.go index 6f4d2b069..4f78b1676 100644 --- a/pkg/deployment/reconcile/action_wait_for_member_up.go +++ b/pkg/deployment/reconcile/action_wait_for_member_up.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" "time" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/errors" driver "github.com/arangodb/go-driver" @@ -75,22 +77,26 @@ func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, bool, a.log.Debug().Msg("Member in failed phase") return true, false, nil } + + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() + if a.action.Group.IsArangosync() { - return a.checkProgressArangoSync(ctx) + return a.checkProgressArangoSync(ctxChild) } switch a.actionCtx.GetMode() { case api.DeploymentModeSingle: - return a.checkProgressSingle(ctx) + return a.checkProgressSingle(ctxChild) case api.DeploymentModeActiveFailover: if a.action.Group == api.ServerGroupAgents { - return a.checkProgressAgent(ctx) + return a.checkProgressAgent(ctxChild) } - return a.checkProgressSingleInActiveFailover(ctx) + return a.checkProgressSingleInActiveFailover(ctxChild) default: if a.action.Group == api.ServerGroupAgents { - return a.checkProgressAgent(ctx) + return a.checkProgressAgent(ctxChild) } - return a.checkProgressCluster(ctx) + return a.checkProgressCluster(ctxChild) } } @@ -98,6 +104,7 @@ func (a *actionWaitForMemberUp) CheckProgress(ctx context.Context) (bool, bool, // of a single server. func (a *actionWaitForMemberUp) checkProgressSingle(ctx context.Context) (bool, bool, error) { log := a.log + c, err := a.actionCtx.GetDatabaseClient(ctx) if err != nil { log.Debug().Err(err).Msg("Failed to create database client") diff --git a/pkg/deployment/reconcile/context.go b/pkg/deployment/reconcile/context.go index 333c0c71e..d00041227 100644 --- a/pkg/deployment/reconcile/context.go +++ b/pkg/deployment/reconcile/context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -25,18 +26,16 @@ package reconcile import ( "context" - inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" - - "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" - - backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" - "github.com/arangodb/arangosync-client/client" driver "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" + v1 "k8s.io/api/core/v1" + + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod/conn" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" - v1 "k8s.io/api/core/v1" + inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" ) // Context provides methods to the reconcile package. @@ -49,9 +48,9 @@ type Context interface { GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error + UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error // UpdateMember updates the deployment status wrt the given member. - UpdateMember(member api.MemberStatus) error + UpdateMember(ctx context.Context, member api.MemberStatus) error // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), // creating one if needed. GetDatabaseClient(ctx context.Context) (driver.Client, error) @@ -70,29 +69,29 @@ type Context interface { // CreateMember adds a new member to the given group. // If ID is non-empty, it will be used, otherwise a new ID is created. // Returns ID, error - CreateMember(group api.ServerGroup, id string) (string, error) + CreateMember(ctx context.Context, group api.ServerGroup, id string) (string, error) // GetPod returns pod. - GetPod(podName string) (*v1.Pod, error) + GetPod(ctx context.Context, podName string) (*v1.Pod, error) // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. - DeletePvc(pvcName string) error + DeletePvc(ctx context.Context, pvcName string) error // RemovePodFinalizers removes all the finalizers from the Pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - RemovePodFinalizers(podName string) error + RemovePodFinalizers(ctx context.Context, podName string) error // UpdatePvc update PVC with given name in the namespace // of the deployment. - UpdatePvc(pvc *v1.PersistentVolumeClaim) error + UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error // GetPvc gets a PVC by the given name, in the samespace of the deployment. - GetPvc(pvcName string) (*v1.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*v1.PersistentVolumeClaim, error) // GetTLSKeyfile returns the keyfile encoded TLS certificate+key for // the given member. GetTLSKeyfile(group api.ServerGroup, member api.MemberStatus) (string, error) // DeleteTLSKeyfile removes the Secret containing the TLS keyfile for the given member. // If the secret does not exist, the error is ignored. - DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error + DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error // DeleteSecret removes the Secret with given name. // If the secret does not exist, the error is ignored. DeleteSecret(secretName string) error @@ -103,21 +102,21 @@ type Context interface { // InvalidateSyncStatus resets the sync state to false and triggers an inspection InvalidateSyncStatus() // DisableScalingCluster disables scaling DBservers and coordinators - DisableScalingCluster() error + DisableScalingCluster(ctx context.Context) error // EnableScalingCluster enables scaling DBservers and coordinators - EnableScalingCluster() error + EnableScalingCluster(ctx context.Context) error // GetAgencyData object for key path GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error // Renders Pod definition for member - RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) + RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*v1.Pod, error) // SelectImage select currently used image by pod SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives deployment name GetName() string // GetAuthentication return authentication for members diff --git a/pkg/deployment/reconcile/helper_shutdown.go b/pkg/deployment/reconcile/helper_shutdown.go index 1ce2951f0..445764ab0 100644 --- a/pkg/deployment/reconcile/helper_shutdown.go +++ b/pkg/deployment/reconcile/helper_shutdown.go @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/rs/zerolog" @@ -60,23 +62,25 @@ func (s shutdownHelperAPI) Start(ctx context.Context) (bool, error) { return true, nil } // Remove finalizers, so Kubernetes will quickly terminate the pod - if err := s.actionCtx.RemovePodFinalizers(m.PodName); err != nil { + if err := s.actionCtx.RemovePodFinalizers(ctx, m.PodName); err != nil { return false, errors.WithStack(err) } if group.IsArangod() { // Invoke shutdown endpoint - c, err := s.actionCtx.GetServerClient(ctx, group, s.action.MemberID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := s.actionCtx.GetServerClient(ctxChild, group, s.action.MemberID) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, errors.WithStack(err) } removeFromCluster := false log.Debug().Bool("removeFromCluster", removeFromCluster).Msg("Shutting down member") - ctx, cancel := context.WithTimeout(ctx, shutdownTimeout) + ctxChild, cancel = context.WithTimeout(ctx, shutdownTimeout) defer cancel() - if err := c.Shutdown(ctx, removeFromCluster); err != nil { + if err := c.Shutdown(ctxChild, removeFromCluster); err != nil { // Shutdown failed. Let's check if we're already done - if ready, _, err := s.CheckProgress(ctx); err == nil && ready { + if ready, _, err := s.CheckProgress(ctxChild); err == nil && ready { // We're done return true, nil } @@ -85,7 +89,7 @@ func (s shutdownHelperAPI) Start(ctx context.Context) (bool, error) { } } else if group.IsArangosync() { // Terminate pod - if err := s.actionCtx.DeletePod(m.PodName); err != nil { + if err := s.actionCtx.DeletePod(ctx, m.PodName); err != nil { return false, errors.WithStack(err) } } @@ -127,7 +131,7 @@ func (s shutdownHelperDelete) Start(ctx context.Context) (bool, error) { } // Terminate pod - if err := s.actionCtx.DeletePod(m.PodName); err != nil { + if err := s.actionCtx.DeletePod(ctx, m.PodName); err != nil { if !k8sutil.IsNotFound(err) { return false, errors.WithStack(err) } @@ -153,7 +157,7 @@ func (s shutdownHelperDelete) CheckProgress(ctx context.Context) (bool, bool, er } if m.PodName != "" { - if _, err := s.actionCtx.GetPod(m.PodName); err == nil { + if _, err := s.actionCtx.GetPod(ctx, m.PodName); err == nil { log.Warn().Msgf("Pod still exists") return false, false, nil } else if !k8sutil.IsNotFound(err) { diff --git a/pkg/deployment/reconcile/plan_builder.go b/pkg/deployment/reconcile/plan_builder.go index 785f12f30..833bbf0ff 100644 --- a/pkg/deployment/reconcile/plan_builder.go +++ b/pkg/deployment/reconcile/plan_builder.go @@ -85,7 +85,7 @@ func (d *Reconciler) CreatePlan(ctx context.Context, cachedStatus inspectorInter status.Plan = newPlan - if err := d.context.UpdateStatus(status, lastVersion); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err), false } return nil, true diff --git a/pkg/deployment/reconcile/plan_builder_cluster.go b/pkg/deployment/reconcile/plan_builder_cluster.go index 44dfffdb0..eef43e29c 100644 --- a/pkg/deployment/reconcile/plan_builder_cluster.go +++ b/pkg/deployment/reconcile/plan_builder_cluster.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -29,6 +30,7 @@ import ( "github.com/arangodb/go-driver" api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" "github.com/rs/zerolog" @@ -39,24 +41,30 @@ const coordinatorHealthFailedTimeout time.Duration = time.Minute func createClusterOperationPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan { if spec.GetMode() != api.DeploymentModeCluster { return nil } - c, err := context.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := planCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { return nil } - cluster, err := c.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cluster, err := c.Cluster(ctxChild) + cancel() if err != nil { log.Warn().Err(err).Msgf("Unable to get Cluster client") return nil } - health, err := cluster.Health(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + health, err := cluster.Health(ctxChild) + cancel() if err != nil { log.Warn().Err(err).Msgf("Unable to get Cluster health") return nil diff --git a/pkg/deployment/reconcile/plan_builder_common.go b/pkg/deployment/reconcile/plan_builder_common.go index da83465f9..fcdbf9373 100644 --- a/pkg/deployment/reconcile/plan_builder_common.go +++ b/pkg/deployment/reconcile/plan_builder_common.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -25,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/deployment/agency" "github.com/arangodb/kube-arangodb/pkg/deployment/features" @@ -36,7 +39,7 @@ import ( func createMaintenanceManagementPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan { if spec.Mode.Get() == api.DeploymentModeSingle { return nil } @@ -46,13 +49,17 @@ func createMaintenanceManagementPlan(ctx context.Context, return nil } - client, err := context.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := planCtx.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get agency client") return nil } - m, err := agency.GetMaintenanceMode(ctx, client) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + m, err := agency.GetMaintenanceMode(ctxChild, client) + cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get agency maintenance mode") return nil diff --git a/pkg/deployment/reconcile/plan_builder_context.go b/pkg/deployment/reconcile/plan_builder_context.go index fc13cc108..2cb8550fe 100644 --- a/pkg/deployment/reconcile/plan_builder_context.go +++ b/pkg/deployment/reconcile/plan_builder_context.go @@ -48,7 +48,7 @@ type PlanBuilderContext interface { // On error, the error is logged. CreateEvent(evt *k8sutil.Event) // GetPvc gets a PVC by the given name, in the samespace of the deployment. - GetPvc(pvcName string) (*core.PersistentVolumeClaim, error) + GetPvc(ctx context.Context, pvcName string) (*core.PersistentVolumeClaim, error) // GetShardSyncStatus returns true if all shards are in sync GetShardSyncStatus() bool // InvalidateSyncStatus resets the sync state to false and triggers an inspection @@ -60,7 +60,7 @@ type PlanBuilderContext interface { // GetAgencyData object for key path GetAgencyData(ctx context.Context, i interface{}, keyParts ...string) error // Renders Pod definition for member - RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) + RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) // SelectImage select currently used image by pod SelectImage(spec api.DeploymentSpec, status api.DeploymentStatus) (api.ImageInfo, bool) // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), @@ -73,7 +73,7 @@ type PlanBuilderContext interface { // SecretsInterface return secret interface SecretsInterface() k8sutil.SecretInterface // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) // GetName receives deployment name GetName() string // GetAgency returns a connection to the entire agency. diff --git a/pkg/deployment/reconcile/plan_builder_encryption.go b/pkg/deployment/reconcile/plan_builder_encryption.go index 00bffb0c8..1dcc68d1d 100644 --- a/pkg/deployment/reconcile/plan_builder_encryption.go +++ b/pkg/deployment/reconcile/plan_builder_encryption.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/deployment/features" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" core "k8s.io/api/core/v1" @@ -269,7 +271,7 @@ func areEncryptionKeysUpToDate(ctx context.Context, func isEncryptionKeyUpToDate(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext, + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext, group api.ServerGroup, m api.MemberStatus, folder *core.Secret) (updateRequired bool, failed bool) { if m.Phase != api.MemberPhaseCreated { @@ -282,7 +284,9 @@ func isEncryptionKeyUpToDate(ctx context.Context, mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger() - c, err := context.GetServerClient(ctx, group, m.ID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := planCtx.GetServerClient(ctxChild, group, m.ID) + cancel() if err != nil { mlog.Warn().Err(err).Msg("Unable to get client") return false, true @@ -290,7 +294,9 @@ func isEncryptionKeyUpToDate(ctx context.Context, client := client.NewClient(c.Connection()) - e, err := client.GetEncryption(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + e, err := client.GetEncryption(ctxChild) + cancel() if err != nil { mlog.Error().Err(err).Msgf("Unable to fetch encryption keys") return false, true diff --git a/pkg/deployment/reconcile/plan_builder_restore.go b/pkg/deployment/reconcile/plan_builder_restore.go index bce1d0c60..2468bd8af 100644 --- a/pkg/deployment/reconcile/plan_builder_restore.go +++ b/pkg/deployment/reconcile/plan_builder_restore.go @@ -49,7 +49,7 @@ func createRestorePlan(ctx context.Context, } if spec.RestoreFrom != nil && status.Restore == nil { - backup, err := context.GetBackup(spec.GetRestoreFrom()) + backup, err := context.GetBackup(ctx, spec.GetRestoreFrom()) if err != nil { log.Warn().Err(err).Msg("Backup not found") return nil @@ -111,7 +111,7 @@ func createRestorePlanEncryption(ctx context.Context, log zerolog.Logger, spec a secret := *spec.RestoreEncryptionSecret // Additional logic to do restore with encryption key - name, _, exists, err := pod.GetEncryptionKey(builderCtx.SecretsInterface(), secret) + name, _, exists, err := pod.GetEncryptionKey(ctx, builderCtx.SecretsInterface(), secret) if err != nil { log.Err(err).Msgf("Unable to fetch encryption key") return false, nil diff --git a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go index 7c49765e5..af4b79d33 100644 --- a/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go +++ b/pkg/deployment/reconcile/plan_builder_rotate_upgrade.go @@ -58,7 +58,7 @@ func createRotateOrUpgradePlan(ctx context.Context, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { var plan api.Plan - newPlan, idle := createRotateOrUpgradePlanInternal(log, apiObject, spec, status, cachedStatus, context) + newPlan, idle := createRotateOrUpgradePlanInternal(ctx, log, apiObject, spec, status, cachedStatus, context) if idle { plan = append(plan, api.NewAction(api.ActionTypeIdle, api.ServerGroupUnknown, "")) @@ -68,7 +68,7 @@ func createRotateOrUpgradePlan(ctx context.Context, return plan } -func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, +func createRotateOrUpgradePlanInternal(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) (api.Plan, bool) { var newPlan api.Plan @@ -116,7 +116,7 @@ func createRotateOrUpgradePlanInternal(log zerolog.Logger, apiObject k8sutil.API !decision.AutoUpgradeNeeded) } else { // Use new level of rotate logic - rotNeeded, reason := podNeedsRotation(log, pod, apiObject, spec, group, status, m, cachedStatus, context) + rotNeeded, reason := podNeedsRotation(ctx, log, pod, apiObject, spec, group, status, m, cachedStatus, context) if rotNeeded { newPlan = createRotateMemberPlan(log, m, group, reason) } @@ -283,9 +283,9 @@ func memberImageInfo(spec api.DeploymentSpec, status api.MemberStatus, images ap // given pod differs from what it should be according to the // given deployment spec. // When true is returned, a reason for the rotation is already returned. -func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object, spec api.DeploymentSpec, +func podNeedsRotation(ctx context.Context, log zerolog.Logger, p *core.Pod, apiObject metav1.Object, spec api.DeploymentSpec, group api.ServerGroup, status api.DeploymentStatus, m api.MemberStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) (bool, string) { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) (bool, string) { if m.PodUID != p.UID { return true, "Pod UID does not match, this pod is not managed by Operator. Recreating" } @@ -294,7 +294,7 @@ func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object, return true, "Pod Spec Version is nil - recreating pod" } - imageInfo, imageFound := context.SelectImage(spec, status) + imageInfo, imageFound := planCtx.SelectImage(spec, status) if !imageFound { // Image is not found, so rotation is not needed return false, "" @@ -306,7 +306,7 @@ func podNeedsRotation(log zerolog.Logger, p *core.Pod, apiObject metav1.Object, groupSpec := spec.GetServerGroupSpec(group) - renderedPod, err := context.RenderPodForMember(cachedStatus, spec, status, m.ID, imageInfo) + renderedPod, err := planCtx.RenderPodForMember(ctx, cachedStatus, spec, status, m.ID, imageInfo) if err != nil { log.Err(err).Msg("Error while rendering pod") return false, "" diff --git a/pkg/deployment/reconcile/plan_builder_test.go b/pkg/deployment/reconcile/plan_builder_test.go index 1bfe9acdb..cd0dd0489 100644 --- a/pkg/deployment/reconcile/plan_builder_test.go +++ b/pkg/deployment/reconcile/plan_builder_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -70,7 +71,7 @@ type testContext struct { RecordedEvent *k8sutil.Event } -func (c *testContext) GetPod(podName string) (*core.Pod, error) { +func (c *testContext) GetPod(_ context.Context, podName string) (*core.Pod, error) { if c.ErrPods != nil { return nil, c.ErrPods } @@ -90,7 +91,7 @@ func (c *testContext) GetAuthentication() conn.Auth { } } -func (c *testContext) RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { +func (c *testContext) RenderPodForMember(_ context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { panic("implement me") } @@ -98,7 +99,7 @@ func (c *testContext) GetName() string { panic("implement me") } -func (c *testContext) GetBackup(backup string) (*backupApi.ArangoBackup, error) { +func (c *testContext) GetBackup(_ context.Context, backup string) (*backupApi.ArangoBackup, error) { panic("implement me") } @@ -106,7 +107,7 @@ func (c *testContext) SecretsInterface() k8sutil.SecretInterface { panic("implement me") } -func (c *testContext) WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error { +func (c *testContext) WithStatusUpdate(_ context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error { panic("implement me") } @@ -114,7 +115,7 @@ func (c *testContext) SelectImage(spec api.DeploymentSpec, status api.Deployment panic("implement me") } -func (c *testContext) UpdatePvc(pvc *core.PersistentVolumeClaim) error { +func (c *testContext) UpdatePvc(_ context.Context, pvc *core.PersistentVolumeClaim) error { panic("implement me") } @@ -137,12 +138,12 @@ func (c *testContext) GetSpec() api.DeploymentSpec { return c.ArangoDeployment.Spec } -func (c *testContext) UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error { +func (c *testContext) UpdateStatus(_ context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error { c.ArangoDeployment.Status = status return nil } -func (c *testContext) UpdateMember(member api.MemberStatus) error { +func (c *testContext) UpdateMember(_ context.Context, member api.MemberStatus) error { panic("implement me") } @@ -166,23 +167,23 @@ func (c *testContext) GetSyncServerClient(ctx context.Context, group api.ServerG panic("implement me") } -func (c *testContext) CreateMember(group api.ServerGroup, id string) (string, error) { +func (c *testContext) CreateMember(_ context.Context, group api.ServerGroup, id string) (string, error) { panic("implement me") } -func (c *testContext) DeletePod(podName string) error { +func (c *testContext) DeletePod(_ context.Context, podName string) error { panic("implement me") } -func (c *testContext) DeletePvc(pvcName string) error { +func (c *testContext) DeletePvc(_ context.Context, pvcName string) error { panic("implement me") } -func (c *testContext) RemovePodFinalizers(podName string) error { +func (c *testContext) RemovePodFinalizers(_ context.Context, podName string) error { panic("implement me") } -func (c *testContext) GetOwnedPods() ([]core.Pod, error) { +func (c *testContext) GetOwnedPods(_ context.Context) ([]core.Pod, error) { if c.ErrPods != nil { return nil, c.ErrPods } @@ -193,7 +194,7 @@ func (c *testContext) GetOwnedPods() ([]core.Pod, error) { return c.Pods, c.ErrPods } -func (c *testContext) DeleteTLSKeyfile(group api.ServerGroup, member api.MemberStatus) error { +func (c *testContext) DeleteTLSKeyfile(_ context.Context, group api.ServerGroup, member api.MemberStatus) error { panic("implement me") } @@ -205,11 +206,11 @@ func (c *testContext) GetDeploymentHealth() (driver.ClusterHealth, error) { panic("implement me") } -func (c *testContext) DisableScalingCluster() error { +func (c *testContext) DisableScalingCluster(_ context.Context) error { panic("implement me") } -func (c *testContext) EnableScalingCluster() error { +func (c *testContext) EnableScalingCluster(_ context.Context) error { panic("implement me") } @@ -232,7 +233,7 @@ func (c *testContext) CreateEvent(evt *k8sutil.Event) { } // GetPvc gets a PVC by the given name, in the samespace of the deployment. -func (c *testContext) GetPvc(pvcName string) (*core.PersistentVolumeClaim, error) { +func (c *testContext) GetPvc(_ context.Context, pvcName string) (*core.PersistentVolumeClaim, error) { return c.PVC, c.PVCErr } diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go index 4aaff3f59..6b38a4350 100644 --- a/pkg/deployment/reconcile/plan_builder_tls_sni.go +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package reconcile @@ -26,6 +27,7 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/deployment/features" + "github.com/arangodb/kube-arangodb/pkg/util/arangod" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" @@ -39,7 +41,7 @@ import ( func createRotateTLSServerSNIPlan(ctx context.Context, log zerolog.Logger, apiObject k8sutil.APIObject, spec api.DeploymentSpec, status api.DeploymentStatus, - cachedStatus inspectorInterface.Inspector, context PlanBuilderContext) api.Plan { + cachedStatus inspectorInterface.Inspector, planCtx PlanBuilderContext) api.Plan { if !spec.TLS.IsSecure() { return nil } @@ -80,13 +82,18 @@ func createRotateTLSServerSNIPlan(ctx context.Context, continue } - c, err := context.GetServerClient(ctx, group, m.ID) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := planCtx.GetServerClient(ctxChild, group, m.ID) + cancel() if err != nil { log.Warn().Err(err).Msg("Unable to get client") continue } - if ok, err := compareTLSSNIConfig(ctx, c.Connection(), fetchedSecrets, false); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + ok, err := compareTLSSNIConfig(ctxChild, c.Connection(), fetchedSecrets, false) + cancel() + if err != nil { log.Warn().Err(err).Msg("SNI compare failed") return nil diff --git a/pkg/deployment/reconcile/plan_executor.go b/pkg/deployment/reconcile/plan_executor.go index 5d4d33e58..4b175f989 100644 --- a/pkg/deployment/reconcile/plan_executor.go +++ b/pkg/deployment/reconcile/plan_executor.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile @@ -70,7 +71,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte log := logContext.Logger() - action := d.createAction(ctx, log, planAction, cachedStatus) + action := d.createAction(log, planAction, cachedStatus) if planAction.StartTime.IsZero() { // Not started yet ready, err := action.Start(ctx) @@ -95,7 +96,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte status.Plan[0].StartTime = &now } // Save plan update - if err := d.context.UpdateStatus(status, lastVersion, true); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion, true); err != nil { log.Debug().Err(err).Msg("Failed to update CR status") return false, errors.WithStack(err) } @@ -120,7 +121,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte status.Plan[0].MemberID = action.MemberID() } // Save plan update - if err := d.context.UpdateStatus(status, lastVersion); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Failed to update CR status") return false, errors.WithStack(err) } @@ -149,7 +150,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte // Replace plan with empty one and save it. status, lastVersion := d.context.GetStatus() status.Plan = api.Plan{} - if err := d.context.UpdateStatus(status, lastVersion); err != nil { + if err := d.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Debug().Err(err).Msg("Failed to update CR status") return false, errors.WithStack(err) } @@ -164,7 +165,7 @@ func (d *Reconciler) ExecutePlan(ctx context.Context, cachedStatus inspectorInte } // createAction create action object based on action type -func (d *Reconciler) createAction(ctx context.Context, log zerolog.Logger, action api.Action, cachedStatus inspectorInterface.Inspector) Action { +func (d *Reconciler) createAction(log zerolog.Logger, action api.Action, cachedStatus inspectorInterface.Inspector) Action { actionCtx := newActionContext(log.With().Str("id", action.ID).Str("type", action.Type.String()).Logger(), d.context, cachedStatus) f, ok := getActionFactory(action.Type) diff --git a/pkg/deployment/reconcile/reconciler.go b/pkg/deployment/reconcile/reconciler.go index aa9ff6e7b..16757d4bc 100644 --- a/pkg/deployment/reconcile/reconciler.go +++ b/pkg/deployment/reconcile/reconciler.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,13 +18,17 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package reconcile import ( - api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" + "context" + "github.com/rs/zerolog" + + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" ) // Reconciler is the service that takes care of bring the a deployment @@ -43,7 +47,7 @@ func NewReconciler(log zerolog.Logger, context Context) *Reconciler { } // CheckDeployment checks for obviously broken things and fixes them immediately -func (r *Reconciler) CheckDeployment() error { +func (r *Reconciler) CheckDeployment(ctx context.Context) error { spec := r.context.GetSpec() status, _ := r.context.GetStatus() @@ -52,19 +56,19 @@ func (r *Reconciler) CheckDeployment() error { if len(status.Members.Coordinators) == 0 { // No more coordinators! Take immediate action r.log.Error().Msg("No Coordinator members! Create one member immediately") - _, err := r.context.CreateMember(api.ServerGroupCoordinators, "") + _, err := r.context.CreateMember(ctx, api.ServerGroupCoordinators, "") if err != nil { return err } } else if status.Members.Coordinators.AllFailed() { r.log.Error().Msg("All coordinators failed - reset") for _, m := range status.Members.Coordinators { - if err := r.context.DeletePod(m.PodName); err != nil { + if err := r.context.DeletePod(ctx, m.PodName); err != nil { r.log.Error().Err(err).Msg("Failed to delete pod") } m.Phase = api.MemberPhaseNone - if err := r.context.UpdateMember(m); err != nil { + if err := r.context.UpdateMember(ctx, m); err != nil { r.log.Error().Err(err).Msg("Failed to update member") } } diff --git a/pkg/deployment/resilience/context.go b/pkg/deployment/resilience/context.go index cc4543894..67ca3fa56 100644 --- a/pkg/deployment/resilience/context.go +++ b/pkg/deployment/resilience/context.go @@ -37,7 +37,7 @@ type Context interface { GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error + UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetAgencyClients returns a client connection for every agency member. // If the given predicate is not nil, only agents are included where the given predicate returns true. GetAgencyClients(ctx context.Context, predicate func(id string) bool) ([]driver.Connection, error) diff --git a/pkg/deployment/resilience/member_failure.go b/pkg/deployment/resilience/member_failure.go index e1339b52e..05c8e39d7 100644 --- a/pkg/deployment/resilience/member_failure.go +++ b/pkg/deployment/resilience/member_failure.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resilience @@ -42,7 +43,7 @@ const ( // CheckMemberFailure performs a check for members that should be in failed state because: // - They are frequently restarted // - They cannot be scheduled for a long time (TODO) -func (r *Resilience) CheckMemberFailure() error { +func (r *Resilience) CheckMemberFailure(ctx context.Context) error { status, lastVersion := r.context.GetStatus() updateStatusNeeded := false if err := status.Members.ForeachServerGroup(func(group api.ServerGroup, list api.MemberStatusList) error { @@ -75,7 +76,8 @@ func (r *Resilience) CheckMemberFailure() error { if !m.Phase.IsFailed() { if m.IsNotReadySince(time.Now().Add(-notReadySinceGracePeriod)) { // Member has terminated too often in recent history. - failureAcceptable, reason, err := r.isMemberFailureAcceptable(status, group, m) + + failureAcceptable, reason, err := r.isMemberFailureAcceptable(ctx, group, m) if err != nil { log.Warn().Err(err).Msg("Failed to check is member failure is acceptable") } else if failureAcceptable { @@ -94,7 +96,7 @@ func (r *Resilience) CheckMemberFailure() error { count := m.RecentTerminationsSince(time.Now().Add(-recentTerminationsSinceGracePeriod)) if count >= recentTerminationThreshold { // Member has terminated too often in recent history. - failureAcceptable, reason, err := r.isMemberFailureAcceptable(status, group, m) + failureAcceptable, reason, err := r.isMemberFailureAcceptable(ctx, group, m) if err != nil { log.Warn().Err(err).Msg("Failed to check is member failure is acceptable") } else if failureAcceptable { @@ -114,7 +116,7 @@ func (r *Resilience) CheckMemberFailure() error { return errors.WithStack(err) } if updateStatusNeeded { - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -125,12 +127,14 @@ func (r *Resilience) CheckMemberFailure() error { // isMemberFailureAcceptable checks if it is currently acceptable to switch the phase of the given member // to failed, which means that it will be replaced. // Return: failureAcceptable, notAcceptableReason, error -func (r *Resilience) isMemberFailureAcceptable(status api.DeploymentStatus, group api.ServerGroup, m api.MemberStatus) (bool, string, error) { - ctx := context.Background() +func (r *Resilience) isMemberFailureAcceptable(ctx context.Context, group api.ServerGroup, m api.MemberStatus) (bool, string, error) { + switch group { case api.ServerGroupAgents: // All good when remaining agents are health - clients, err := r.context.GetAgencyClients(ctx, func(id string) bool { return id != m.ID }) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + clients, err := r.context.GetAgencyClients(ctxChild, func(id string) bool { return id != m.ID }) + cancel() if err != nil { return false, "", errors.WithStack(err) } @@ -139,7 +143,9 @@ func (r *Resilience) isMemberFailureAcceptable(status api.DeploymentStatus, grou } return true, "", nil case api.ServerGroupDBServers: - client, err := r.context.GetDatabaseClient(ctx) + ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + client, err := r.context.GetDatabaseClient(ctxChild) + cancel() if err != nil { return false, "", errors.WithStack(err) } diff --git a/pkg/deployment/resources/annotations.go b/pkg/deployment/resources/annotations.go index 471e50f8a..e87701d44 100644 --- a/pkg/deployment/resources/annotations.go +++ b/pkg/deployment/resources/annotations.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package resources @@ -29,7 +30,6 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/collection" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - monitoringTypedClient "github.com/prometheus-operator/prometheus-operator/pkg/client/versioned/typed/monitoring/v1" "k8s.io/apimachinery/pkg/types" "github.com/arangodb/kube-arangodb/pkg/apis/deployment" @@ -39,17 +39,26 @@ import ( core "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" - typedCore "k8s.io/client-go/kubernetes/typed/core/v1" - policyTyped "k8s.io/client-go/kubernetes/typed/policy/v1beta1" ) -func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) error { +type PatchFunc func(name string, d []byte) error + +func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { kubecli := r.context.GetKubeCli() monitoringcli := r.context.GetMonitoringV1Cli() log.Info().Msgf("Ensuring annotations") - if err := ensureSecretsAnnotations(kubecli.CoreV1().Secrets(r.context.GetNamespace()), + patchSecret := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := kubecli.CoreV1().Secrets(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + } + + if err := ensureSecretsAnnotations(patchSecret, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -58,7 +67,16 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensureServiceAccountsAnnotations(kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()), + patchServiceAccount := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + } + + if err := ensureServiceAccountsAnnotations(patchServiceAccount, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -67,7 +85,16 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensureServicesAnnotations(kubecli.CoreV1().Services(r.context.GetNamespace()), + patchService := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := kubecli.CoreV1().Services(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + } + + if err := ensureServicesAnnotations(patchService, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -76,7 +103,16 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensurePdbsAnnotations(kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()), + patchPDB := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + } + + if err := ensurePdbsAnnotations(patchPDB, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -85,7 +121,16 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensurePvcsAnnotations(kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()), + patchPVC := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + } + + if err := ensurePvcsAnnotations(patchPVC, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -94,7 +139,16 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensurePodsAnnotations(kubecli.CoreV1().Pods(r.context.GetNamespace()), + patchPod := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := kubecli.CoreV1().Pods(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + } + + if err := ensurePodsAnnotations(patchPod, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -104,7 +158,16 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return err } - if err := ensureServiceMonitorsAnnotations(monitoringcli.ServiceMonitors(r.context.GetNamespace()), + patchServiceMonitor := func(name string, d []byte) error { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := monitoringcli.ServiceMonitors(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + } + + if err := ensureServiceMonitorsAnnotations(patchServiceMonitor, cachedStatus, deployment.ArangoDeploymentResourceKind, r.context.GetAPIObject().GetName(), @@ -116,12 +179,9 @@ func (r *Resources) EnsureAnnotations(cachedStatus inspectorInterface.Inspector) return nil } -func ensureSecretsAnnotations(client typedCore.SecretInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureSecretsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateSecrets(func(secret *core.Secret) error { - ensureAnnotationsMap(secret.Kind, secret, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(secret.Kind, secret, spec, patch) return nil }, func(secret *core.Secret) bool { return k8sutil.IsChildResource(kind, name, namespace, secret) @@ -132,12 +192,9 @@ func ensureSecretsAnnotations(client typedCore.SecretInterface, cachedStatus ins return nil } -func ensureServiceAccountsAnnotations(client typedCore.ServiceAccountInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureServiceAccountsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateServiceAccounts(func(serviceAccount *core.ServiceAccount) error { - ensureAnnotationsMap(serviceAccount.Kind, serviceAccount, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(serviceAccount.Kind, serviceAccount, spec, patch) return nil }, func(serviceAccount *core.ServiceAccount) bool { return k8sutil.IsChildResource(kind, name, namespace, serviceAccount) @@ -148,12 +205,9 @@ func ensureServiceAccountsAnnotations(client typedCore.ServiceAccountInterface, return nil } -func ensureServicesAnnotations(client typedCore.ServiceInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureServicesAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateServices(func(service *core.Service) error { - ensureAnnotationsMap(service.Kind, service, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(service.Kind, service, spec, patch) return nil }, func(service *core.Service) bool { return k8sutil.IsChildResource(kind, name, namespace, service) @@ -164,12 +218,9 @@ func ensureServicesAnnotations(client typedCore.ServiceInterface, cachedStatus i return nil } -func ensurePdbsAnnotations(client policyTyped.PodDisruptionBudgetInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensurePdbsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IteratePodDisruptionBudgets(func(podDisruptionBudget *policy.PodDisruptionBudget) error { - ensureAnnotationsMap(podDisruptionBudget.Kind, podDisruptionBudget, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(podDisruptionBudget.Kind, podDisruptionBudget, spec, patch) return nil }, func(podDisruptionBudget *policy.PodDisruptionBudget) bool { return k8sutil.IsChildResource(kind, name, namespace, podDisruptionBudget) @@ -180,12 +231,9 @@ func ensurePdbsAnnotations(client policyTyped.PodDisruptionBudgetInterface, cach return nil } -func ensurePvcsAnnotations(client typedCore.PersistentVolumeClaimInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensurePvcsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IteratePersistentVolumeClaims(func(persistentVolumeClaim *core.PersistentVolumeClaim) error { - ensureGroupAnnotationsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureGroupAnnotationsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, spec, patch) return nil }, func(persistentVolumeClaim *core.PersistentVolumeClaim) bool { return k8sutil.IsChildResource(kind, name, namespace, persistentVolumeClaim) @@ -196,12 +244,9 @@ func ensurePvcsAnnotations(client typedCore.PersistentVolumeClaimInterface, cach return nil } -func ensureServiceMonitorsAnnotations(client monitoringTypedClient.ServiceMonitorInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { +func ensureServiceMonitorsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, spec api.DeploymentSpec) error { if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error { - ensureAnnotationsMap(serviceMonitor.Kind, serviceMonitor, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureAnnotationsMap(serviceMonitor.Kind, serviceMonitor, spec, patch) return nil }, func(serviceMonitor *monitoring.ServiceMonitor) bool { return k8sutil.IsChildResource(kind, name, namespace, serviceMonitor) @@ -226,12 +271,9 @@ func getObjectGroup(obj meta.Object) api.ServerGroup { return api.ServerGroupFromRole(group) } -func ensurePodsAnnotations(client typedCore.PodInterface, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, annotations map[string]string, spec api.DeploymentSpec) error { +func ensurePodsAnnotations(patch PatchFunc, cachedStatus inspectorInterface.Inspector, kind, name, namespace string, annotations map[string]string, spec api.DeploymentSpec) error { if err := cachedStatus.IteratePods(func(pod *core.Pod) error { - ensureGroupAnnotationsMap(pod.Kind, pod, spec, func(name string, d []byte) error { - _, err := client.Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) - return err - }) + ensureGroupAnnotationsMap(pod.Kind, pod, spec, patch) return nil }, func(pod *core.Pod) bool { return k8sutil.IsChildResource(kind, name, namespace, pod) @@ -292,8 +334,7 @@ func ensureGroupAnnotationsMap(kind string, obj meta.Object, spec api.Deployment return ensureObjectMap(kind, obj, mode, expected, obj.GetAnnotations(), collection.AnnotationsPatch, patchCmd, ignoredList...) } -func ensureAnnotationsMap(kind string, obj meta.Object, spec api.DeploymentSpec, - patchCmd func(name string, d []byte) error) bool { +func ensureAnnotationsMap(kind string, obj meta.Object, spec api.DeploymentSpec, patchCmd PatchFunc) bool { expected := spec.Annotations ignored := spec.AnnotationsIgnoreList @@ -305,7 +346,7 @@ func ensureAnnotationsMap(kind string, obj meta.Object, spec api.DeploymentSpec, func ensureObjectMap(kind string, obj meta.Object, mode api.LabelsMode, expected, actual map[string]string, patchGetter func(mode api.LabelsMode, expected map[string]string, actual map[string]string, ignored ...string) patch.Patch, - patchCmd func(name string, d []byte) error, + patchCmd PatchFunc, ignored ...string) bool { p := patchGetter(mode, expected, actual, ignored...) diff --git a/pkg/deployment/resources/certificates_client_auth.go b/pkg/deployment/resources/certificates_client_auth.go index 37d23acae..2b7cfd3e6 100644 --- a/pkg/deployment/resources/certificates_client_auth.go +++ b/pkg/deployment/resources/certificates_client_auth.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" "fmt" "strings" "time" @@ -44,7 +46,7 @@ const ( // createClientAuthCACertificate creates a client authentication CA certificate and stores it in a secret with name // specified in the given spec. -func createClientAuthCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec, deploymentName string, ownerRef *metav1.OwnerReference) error { +func createClientAuthCACertificate(ctx context.Context, log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec, deploymentName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", spec.GetClientCASecretName()).Logger() options := certificates.CreateCertificateOptions{ CommonName: fmt.Sprintf("%s Client Authentication Root Certificate", deploymentName), @@ -59,7 +61,7 @@ func createClientAuthCACertificate(log zerolog.Logger, secrets k8sutil.SecretInt log.Debug().Err(err).Msg("Failed to create CA certificate") return errors.WithStack(err) } - if err := k8sutil.CreateCASecret(secrets, spec.GetClientCASecretName(), cert, priv, ownerRef); err != nil { + if err := k8sutil.CreateCASecret(ctx, secrets, spec.GetClientCASecretName(), cert, priv, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("CA Secret already exists") } else { @@ -76,7 +78,7 @@ func createClientAuthCACertificate(log zerolog.Logger, secrets k8sutil.SecretInt func createClientAuthCertificateKeyfile(log zerolog.Logger, secrets v1.SecretInterface, commonName string, ttl time.Duration, spec api.SyncAuthenticationSpec, secretName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", secretName).Logger() // Load CA certificate - caCert, caKey, _, err := k8sutil.GetCASecret(secrets, spec.GetClientCASecretName(), nil) + caCert, caKey, _, err := k8sutil.GetCASecret(context.TODO(), secrets, spec.GetClientCASecretName(), nil) if err != nil { log.Debug().Err(err).Msg("Failed to load CA certificate") return errors.WithStack(err) @@ -102,7 +104,7 @@ func createClientAuthCertificateKeyfile(log zerolog.Logger, secrets v1.SecretInt } keyfile := strings.TrimSpace(cert) + "\n" + strings.TrimSpace(priv) - if err := k8sutil.CreateTLSKeyfileSecret(secrets, secretName, keyfile, ownerRef); err != nil { + if err := k8sutil.CreateTLSKeyfileSecret(context.TODO(), secrets, secretName, keyfile, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("Server Secret already exists") } else { diff --git a/pkg/deployment/resources/certificates_tls.go b/pkg/deployment/resources/certificates_tls.go index 73913af1a..be0ff2b0a 100644 --- a/pkg/deployment/resources/certificates_tls.go +++ b/pkg/deployment/resources/certificates_tls.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" "fmt" "strings" "time" @@ -45,7 +47,8 @@ const ( // createTLSCACertificate creates a CA certificate and stores it in a secret with name // specified in the given spec. -func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.TLSSpec, deploymentName string, ownerRef *metav1.OwnerReference) error { +func createTLSCACertificate(ctx context.Context, log zerolog.Logger, secrets k8sutil.SecretInterface, spec api.TLSSpec, + deploymentName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", spec.GetCASecretName()).Logger() options := certificates.CreateCertificateOptions{ @@ -60,7 +63,7 @@ func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, log.Debug().Err(err).Msg("Failed to create CA certificate") return errors.WithStack(err) } - if err := k8sutil.CreateCASecret(secrets, spec.GetCASecretName(), cert, priv, ownerRef); err != nil { + if err := k8sutil.CreateCASecret(ctx, secrets, spec.GetCASecretName(), cert, priv, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("CA Secret already exists") } else { @@ -74,7 +77,7 @@ func createTLSCACertificate(log zerolog.Logger, secrets k8sutil.SecretInterface, // createTLSServerCertificate creates a TLS certificate for a specific server and stores // it in a secret with the given name. -func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, serverNames []string, spec api.TLSSpec, +func createTLSServerCertificate(ctx context.Context, log zerolog.Logger, secrets v1.SecretInterface, serverNames []string, spec api.TLSSpec, secretName string, ownerRef *metav1.OwnerReference) error { log = log.With().Str("secret", secretName).Logger() @@ -86,7 +89,9 @@ func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, } // Load CA certificate - caCert, caKey, _, err := k8sutil.GetCASecret(secrets, spec.GetCASecretName(), nil) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + caCert, caKey, _, err := k8sutil.GetCASecret(ctxChild, secrets, spec.GetCASecretName(), nil) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to load CA certificate") return errors.WithStack(err) @@ -113,7 +118,10 @@ func createTLSServerCertificate(log zerolog.Logger, secrets v1.SecretInterface, } keyfile := strings.TrimSpace(cert) + "\n" + strings.TrimSpace(priv) - if err := k8sutil.CreateTLSKeyfileSecret(secrets, secretName, keyfile, ownerRef); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + if err := k8sutil.CreateTLSKeyfileSecret(ctxChild, secrets, secretName, keyfile, ownerRef); err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("Server Secret already exists") } else { diff --git a/pkg/deployment/resources/context.go b/pkg/deployment/resources/context.go index 6da4c6e8d..2f4ad19aa 100644 --- a/pkg/deployment/resources/context.go +++ b/pkg/deployment/resources/context.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -65,7 +66,7 @@ type Context interface { GetStatus() (api.DeploymentStatus, int32) // UpdateStatus replaces the status of the deployment with the given status and // updates the resources in k8s. - UpdateStatus(status api.DeploymentStatus, lastVersion int32, force ...bool) error + UpdateStatus(ctx context.Context, status api.DeploymentStatus, lastVersion int32, force ...bool) error // GetKubeCli returns the kubernetes client GetKubeCli() kubernetes.Interface // GetMonitoringV1Cli returns monitoring client @@ -89,13 +90,13 @@ type Context interface { GetOwnedPVCs() ([]v1.PersistentVolumeClaim, error) // CleanupPod deletes a given pod with force and explicit UID. // If the pod does not exist, the error is ignored. - CleanupPod(p *v1.Pod) error + CleanupPod(ctx context.Context, p *v1.Pod) error // DeletePod deletes a pod with given name in the namespace // of the deployment. If the pod does not exist, the error is ignored. - DeletePod(podName string) error + DeletePod(ctx context.Context, podName string) error // DeletePvc deletes a persistent volume claim with given name in the namespace // of the deployment. If the pvc does not exist, the error is ignored. - DeletePvc(pvcName string) error + DeletePvc(ctx context.Context, pvcName string) error // GetAgencyClients returns a client connection for every agency member. GetAgencyClients(ctx context.Context, predicate func(memberID string) bool) ([]driver.Connection, error) // GetDatabaseClient returns a cached client for the entire database (cluster coordinators or single server), @@ -104,9 +105,9 @@ type Context interface { // GetAgency returns a connection to the entire agency. GetAgency(ctx context.Context) (agency.Agency, error) // WithStatusUpdate update status of ArangoDeployment with defined modifier. If action returns True action is taken - WithStatusUpdate(action func(s *api.DeploymentStatus) bool, force ...bool) error + WithStatusUpdate(ctx context.Context, action func(s *api.DeploymentStatus) bool, force ...bool) error // GetBackup receives information about a backup resource - GetBackup(backup string) (*backupApi.ArangoBackup, error) + GetBackup(ctx context.Context, backup string) (*backupApi.ArangoBackup, error) GetScope() scope.Scope GetCachedStatus() inspectorInterface.Inspector diff --git a/pkg/deployment/resources/inspector/inspector.go b/pkg/deployment/resources/inspector/inspector.go index f91dffc15..b1b428716 100644 --- a/pkg/deployment/resources/inspector/inspector.go +++ b/pkg/deployment/resources/inspector/inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -46,42 +47,43 @@ type SecretReadInterface interface { } func NewInspector(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) (inspectorInterface.Inspector, error) { - pods, err := podsToMap(k, namespace) + ctx := context.TODO() + pods, err := podsToMap(ctx, k, namespace) if err != nil { return nil, err } - secrets, err := secretsToMap(k, namespace) + secrets, err := secretsToMap(ctx, k, namespace) if err != nil { return nil, err } - pvcs, err := pvcsToMap(k, namespace) + pvcs, err := pvcsToMap(ctx, k, namespace) if err != nil { return nil, err } - services, err := servicesToMap(k, namespace) + services, err := servicesToMap(ctx, k, namespace) if err != nil { return nil, err } - serviceAccounts, err := serviceAccountsToMap(k, namespace) + serviceAccounts, err := serviceAccountsToMap(ctx, k, namespace) if err != nil { return nil, err } - podDisruptionBudgets, err := podDisruptionBudgetsToMap(k, namespace) + podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, k, namespace) if err != nil { return nil, err } - serviceMonitors, err := serviceMonitorsToMap(m, namespace) + serviceMonitors, err := serviceMonitorsToMap(ctx, m, namespace) if err != nil { return nil, err } - arangoMembers, err := arangoMembersToMap(c, namespace) + arangoMembers, err := arangoMembersToMap(ctx, c, namespace) if err != nil { return nil, err } @@ -130,46 +132,47 @@ type inspector struct { m monitoringClient.MonitoringV1Interface } -func (i *inspector) Refresh(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) error { +func (i *inspector) Refresh(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, + c versioned.Interface, namespace string) error { i.lock.Lock() defer i.lock.Unlock() - pods, err := podsToMap(k, namespace) + pods, err := podsToMap(ctx, k, namespace) if err != nil { return err } - secrets, err := secretsToMap(k, namespace) + secrets, err := secretsToMap(ctx, k, namespace) if err != nil { return err } - pvcs, err := pvcsToMap(k, namespace) + pvcs, err := pvcsToMap(ctx, k, namespace) if err != nil { return err } - services, err := servicesToMap(k, namespace) + services, err := servicesToMap(ctx, k, namespace) if err != nil { return err } - serviceAccounts, err := serviceAccountsToMap(k, namespace) + serviceAccounts, err := serviceAccountsToMap(ctx, k, namespace) if err != nil { return err } - podDisruptionBudgets, err := podDisruptionBudgetsToMap(k, namespace) + podDisruptionBudgets, err := podDisruptionBudgetsToMap(ctx, k, namespace) if err != nil { return err } - serviceMonitors, err := serviceMonitorsToMap(m, namespace) + serviceMonitors, err := serviceMonitorsToMap(ctx, m, namespace) if err != nil { return err } - arangoMembers, err := arangoMembersToMap(c, namespace) + arangoMembers, err := arangoMembersToMap(ctx, c, namespace) if err != nil { return err } diff --git a/pkg/deployment/resources/inspector/members.go b/pkg/deployment/resources/inspector/members.go index 620b45679..7ca95dcca 100644 --- a/pkg/deployment/resources/inspector/members.go +++ b/pkg/deployment/resources/inspector/members.go @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -75,8 +78,8 @@ func (i *inspector) ArangoMember(name string) (*api.ArangoMember, bool) { return arangoMember, true } -func arangoMembersToMap(k versioned.Interface, namespace string) (map[string]*api.ArangoMember, error) { - arangoMembers, err := getArangoMembers(k, namespace, "") +func arangoMembersToMap(ctx context.Context, k versioned.Interface, namespace string) (map[string]*api.ArangoMember, error) { + arangoMembers, err := getArangoMembers(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,18 +102,20 @@ func arangoMemberPointer(pod api.ArangoMember) *api.ArangoMember { return &pod } -func getArangoMembers(k versioned.Interface, namespace, cont string) ([]api.ArangoMember, error) { - arangoMembers, err := k.DatabaseV1().ArangoMembers(namespace).List(context.Background(), meta.ListOptions{ +func getArangoMembers(ctx context.Context, k versioned.Interface, namespace, cont string) ([]api.ArangoMember, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + arangoMembers, err := k.DatabaseV1().ArangoMembers(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if arangoMembers.Continue != "" { - nextArangoMembersLayer, err := getArangoMembers(k, namespace, arangoMembers.Continue) + nextArangoMembersLayer, err := getArangoMembers(ctx, k, namespace, arangoMembers.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/pdbs.go b/pkg/deployment/resources/inspector/pdbs.go index 95bc74b0f..6a1a7727e 100644 --- a/pkg/deployment/resources/inspector/pdbs.go +++ b/pkg/deployment/resources/inspector/pdbs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/poddisruptionbudget" policy "k8s.io/api/policy/v1beta1" @@ -75,8 +78,8 @@ func (i *inspector) PodDisruptionBudget(name string) (*policy.PodDisruptionBudge return podDisruptionBudget, true } -func podDisruptionBudgetsToMap(k kubernetes.Interface, namespace string) (map[string]*policy.PodDisruptionBudget, error) { - podDisruptionBudgets, err := getPodDisruptionBudgets(k, namespace, "") +func podDisruptionBudgetsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*policy.PodDisruptionBudget, error) { + podDisruptionBudgets, err := getPodDisruptionBudgets(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,18 +102,20 @@ func podDisruptionBudgetPointer(podDisruptionBudget policy.PodDisruptionBudget) return &podDisruptionBudget } -func getPodDisruptionBudgets(k kubernetes.Interface, namespace, cont string) ([]policy.PodDisruptionBudget, error) { - podDisruptionBudgets, err := k.PolicyV1beta1().PodDisruptionBudgets(namespace).List(context.Background(), meta.ListOptions{ +func getPodDisruptionBudgets(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]policy.PodDisruptionBudget, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + podDisruptionBudgets, err := k.PolicyV1beta1().PodDisruptionBudgets(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if podDisruptionBudgets.Continue != "" { - nextPodDisruptionBudgetsLayer, err := getPodDisruptionBudgets(k, namespace, podDisruptionBudgets.Continue) + nextPodDisruptionBudgetsLayer, err := getPodDisruptionBudgets(ctx, k, namespace, podDisruptionBudgets.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/pods.go b/pkg/deployment/resources/inspector/pods.go index aad1db3a3..ba2c511d7 100644 --- a/pkg/deployment/resources/inspector/pods.go +++ b/pkg/deployment/resources/inspector/pods.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -26,6 +27,7 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/pod" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -75,8 +77,8 @@ func (i *inspector) Pod(name string) (*core.Pod, bool) { return pod, true } -func podsToMap(k kubernetes.Interface, namespace string) (map[string]*core.Pod, error) { - pods, err := getPods(k, namespace, "") +func podsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.Pod, error) { + pods, err := getPods(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,18 +101,21 @@ func podPointer(pod core.Pod) *core.Pod { return &pod } -func getPods(k kubernetes.Interface, namespace, cont string) ([]core.Pod, error) { - pods, err := k.CoreV1().Pods(namespace).List(context.Background(), meta.ListOptions{ +func getPods(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Pod, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + pods, err := k.CoreV1().Pods(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if pods.Continue != "" { - nextPodsLayer, err := getPods(k, namespace, pods.Continue) + // pass the original context + nextPodsLayer, err := getPods(ctx, k, namespace, pods.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/pvcs.go b/pkg/deployment/resources/inspector/pvcs.go index 10d71493c..dae5b2000 100644 --- a/pkg/deployment/resources/inspector/pvcs.go +++ b/pkg/deployment/resources/inspector/pvcs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" core "k8s.io/api/core/v1" @@ -75,8 +78,8 @@ func (i *inspector) PersistentVolumeClaim(name string) (*core.PersistentVolumeCl return pvc, true } -func pvcsToMap(k kubernetes.Interface, namespace string) (map[string]*core.PersistentVolumeClaim, error) { - pvcs, err := getPersistentVolumeClaims(k, namespace, "") +func pvcsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.PersistentVolumeClaim, error) { + pvcs, err := getPersistentVolumeClaims(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,18 +102,20 @@ func pvcPointer(pvc core.PersistentVolumeClaim) *core.PersistentVolumeClaim { return &pvc } -func getPersistentVolumeClaims(k kubernetes.Interface, namespace, cont string) ([]core.PersistentVolumeClaim, error) { - pvcs, err := k.CoreV1().PersistentVolumeClaims(namespace).List(context.Background(), meta.ListOptions{ +func getPersistentVolumeClaims(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.PersistentVolumeClaim, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + pvcs, err := k.CoreV1().PersistentVolumeClaims(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if pvcs.Continue != "" { - nextPersistentVolumeClaimsLayer, err := getPersistentVolumeClaims(k, namespace, pvcs.Continue) + nextPersistentVolumeClaimsLayer, err := getPersistentVolumeClaims(ctx, k, namespace, pvcs.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/sa.go b/pkg/deployment/resources/inspector/sa.go index 514727d01..aac7e4a11 100644 --- a/pkg/deployment/resources/inspector/sa.go +++ b/pkg/deployment/resources/inspector/sa.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/serviceaccount" core "k8s.io/api/core/v1" @@ -75,8 +78,8 @@ func (i *inspector) ServiceAccount(name string) (*core.ServiceAccount, bool) { return serviceAccount, true } -func serviceAccountsToMap(k kubernetes.Interface, namespace string) (map[string]*core.ServiceAccount, error) { - serviceAccounts, err := getServiceAccounts(k, namespace, "") +func serviceAccountsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.ServiceAccount, error) { + serviceAccounts, err := getServiceAccounts(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,18 +102,20 @@ func serviceAccountPointer(serviceAccount core.ServiceAccount) *core.ServiceAcco return &serviceAccount } -func getServiceAccounts(k kubernetes.Interface, namespace, cont string) ([]core.ServiceAccount, error) { - serviceAccounts, err := k.CoreV1().ServiceAccounts(namespace).List(context.Background(), meta.ListOptions{ +func getServiceAccounts(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.ServiceAccount, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + serviceAccounts, err := k.CoreV1().ServiceAccounts(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if serviceAccounts.Continue != "" { - nextServiceAccountsLayer, err := getServiceAccounts(k, namespace, serviceAccounts.Continue) + nextServiceAccountsLayer, err := getServiceAccounts(ctx, k, namespace, serviceAccounts.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/secrets.go b/pkg/deployment/resources/inspector/secrets.go index d05502dcb..4450d7125 100644 --- a/pkg/deployment/resources/inspector/secrets.go +++ b/pkg/deployment/resources/inspector/secrets.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/secret" core "k8s.io/api/core/v1" @@ -96,8 +99,8 @@ func (s secretReadInterface) Get(ctx context.Context, name string, opts meta.Get } } -func secretsToMap(k kubernetes.Interface, namespace string) (map[string]*core.Secret, error) { - secrets, err := getSecrets(k, namespace, "") +func secretsToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.Secret, error) { + secrets, err := getSecrets(ctx, k, namespace, "") if err != nil { return nil, err } @@ -120,18 +123,20 @@ func secretPointer(pod core.Secret) *core.Secret { return &pod } -func getSecrets(k kubernetes.Interface, namespace, cont string) ([]core.Secret, error) { - secrets, err := k.CoreV1().Secrets(namespace).List(context.Background(), meta.ListOptions{ +func getSecrets(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Secret, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + secrets, err := k.CoreV1().Secrets(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if secrets.Continue != "" { - nextSecretsLayer, err := getSecrets(k, namespace, secrets.Continue) + nextSecretsLayer, err := getSecrets(ctx, k, namespace, secrets.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/services.go b/pkg/deployment/resources/inspector/services.go index 410a65a62..5ccfc85b5 100644 --- a/pkg/deployment/resources/inspector/services.go +++ b/pkg/deployment/resources/inspector/services.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/service" core "k8s.io/api/core/v1" @@ -75,8 +78,8 @@ func (i *inspector) Service(name string) (*core.Service, bool) { return service, true } -func servicesToMap(k kubernetes.Interface, namespace string) (map[string]*core.Service, error) { - services, err := getServices(k, namespace, "") +func servicesToMap(ctx context.Context, k kubernetes.Interface, namespace string) (map[string]*core.Service, error) { + services, err := getServices(ctx, k, namespace, "") if err != nil { return nil, err } @@ -99,18 +102,20 @@ func servicePointer(pod core.Service) *core.Service { return &pod } -func getServices(k kubernetes.Interface, namespace, cont string) ([]core.Service, error) { - services, err := k.CoreV1().Services(namespace).List(context.Background(), meta.ListOptions{ +func getServices(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Service, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + services, err := k.CoreV1().Services(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return nil, err } if services.Continue != "" { - nextServicesLayer, err := getServices(k, namespace, services.Continue) + nextServicesLayer, err := getServices(ctx, k, namespace, services.Continue) if err != nil { return nil, err } diff --git a/pkg/deployment/resources/inspector/sms.go b/pkg/deployment/resources/inspector/sms.go index 3f1d4bbc8..381d27f15 100644 --- a/pkg/deployment/resources/inspector/sms.go +++ b/pkg/deployment/resources/inspector/sms.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector @@ -25,6 +26,8 @@ package inspector import ( "context" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" + "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/servicemonitor" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" @@ -75,8 +78,8 @@ func (i *inspector) ServiceMonitor(name string) (*monitoring.ServiceMonitor, boo return serviceMonitor, true } -func serviceMonitorsToMap(m monitoringClient.MonitoringV1Interface, namespace string) (map[string]*monitoring.ServiceMonitor, error) { - serviceMonitors, err := getServiceMonitors(m, namespace, "") +func serviceMonitorsToMap(ctx context.Context, m monitoringClient.MonitoringV1Interface, namespace string) (map[string]*monitoring.ServiceMonitor, error) { + serviceMonitors, err := getServiceMonitors(ctx, m, namespace, "") if err != nil { return nil, err } @@ -95,11 +98,13 @@ func serviceMonitorsToMap(m monitoringClient.MonitoringV1Interface, namespace st return serviceMonitorMap, nil } -func getServiceMonitors(m monitoringClient.MonitoringV1Interface, namespace, cont string) ([]*monitoring.ServiceMonitor, error) { - serviceMonitors, err := m.ServiceMonitors(namespace).List(context.Background(), meta.ListOptions{ +func getServiceMonitors(ctx context.Context, m monitoringClient.MonitoringV1Interface, namespace, cont string) ([]*monitoring.ServiceMonitor, error) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + serviceMonitors, err := m.ServiceMonitors(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) + cancel() if err != nil { return []*monitoring.ServiceMonitor{}, nil diff --git a/pkg/deployment/resources/labels.go b/pkg/deployment/resources/labels.go index 2c8bfe58d..8d5b45b6b 100644 --- a/pkg/deployment/resources/labels.go +++ b/pkg/deployment/resources/labels.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package resources @@ -26,54 +27,58 @@ import ( "context" "github.com/arangodb/kube-arangodb/pkg/util/errors" + "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" inspectorInterface "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector" monitoring "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - core "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" ) -func (r *Resources) EnsureLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { r.log.Info().Msgf("Ensuring labels") - if err := r.EnsureSecretLabels(cachedStatus); err != nil { + if err := r.EnsureSecretLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsureServiceAccountsLabels(cachedStatus); err != nil { + if err := r.EnsureServiceAccountsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsureServicesLabels(cachedStatus); err != nil { + if err := r.EnsureServicesLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsureServiceMonitorsLabels(cachedStatus); err != nil { + if err := r.EnsureServiceMonitorsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsurePodsLabels(cachedStatus); err != nil { + if err := r.EnsurePodsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsurePersistentVolumeClaimsLabels(cachedStatus); err != nil { + if err := r.EnsurePersistentVolumeClaimsLabels(ctx, cachedStatus); err != nil { return err } - if err := r.EnsurePodDisruptionBudgetsLabels(cachedStatus); err != nil { + if err := r.EnsurePodDisruptionBudgetsLabels(ctx, cachedStatus); err != nil { return err } return nil } -func (r *Resources) EnsureSecretLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureSecretLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateSecrets(func(secret *core.Secret) error { if ensureLabelsMap(secret.Kind, secret, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true @@ -93,11 +98,15 @@ func (r *Resources) EnsureSecretLabels(cachedStatus inspectorInterface.Inspector return nil } -func (r *Resources) EnsureServiceAccountsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServiceAccountsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateServiceAccounts(func(serviceAccount *core.ServiceAccount) error { if ensureLabelsMap(serviceAccount.Kind, serviceAccount, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().ServiceAccounts(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetKubeCli().CoreV1().ServiceAccounts(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true @@ -117,11 +126,15 @@ func (r *Resources) EnsureServiceAccountsLabels(cachedStatus inspectorInterface. return nil } -func (r *Resources) EnsureServicesLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServicesLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateServices(func(service *core.Service) error { if ensureLabelsMap(service.Kind, service, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().Services(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetKubeCli().CoreV1().Services(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true @@ -141,11 +154,15 @@ func (r *Resources) EnsureServicesLabels(cachedStatus inspectorInterface.Inspect return nil } -func (r *Resources) EnsureServiceMonitorsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServiceMonitorsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error { if ensureLabelsMap(serviceMonitor.Kind, serviceMonitor, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true @@ -165,11 +182,15 @@ func (r *Resources) EnsureServiceMonitorsLabels(cachedStatus inspectorInterface. return nil } -func (r *Resources) EnsurePodsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePodsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IteratePods(func(pod *core.Pod) error { if ensureGroupLabelsMap(pod.Kind, pod, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().Pods(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetKubeCli().CoreV1().Pods(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true @@ -189,11 +210,15 @@ func (r *Resources) EnsurePodsLabels(cachedStatus inspectorInterface.Inspector) return nil } -func (r *Resources) EnsurePersistentVolumeClaimsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePersistentVolumeClaimsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IteratePersistentVolumeClaims(func(persistentVolumeClaim *core.PersistentVolumeClaim) error { if ensureGroupLabelsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true @@ -213,11 +238,15 @@ func (r *Resources) EnsurePersistentVolumeClaimsLabels(cachedStatus inspectorInt return nil } -func (r *Resources) EnsurePodDisruptionBudgetsLabels(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePodDisruptionBudgetsLabels(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { changed := false if err := cachedStatus.IteratePodDisruptionBudgets(func(budget *policy.PodDisruptionBudget) error { if ensureLabelsMap(budget.Kind, budget, r.context.GetSpec(), func(name string, d []byte) error { - _, err := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetAPIObject().GetNamespace()).Patch(context.Background(), name, types.JSONPatchType, d, meta.PatchOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() + + _, err := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetAPIObject(). + GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) return err }) { changed = true diff --git a/pkg/deployment/resources/member_cleanup.go b/pkg/deployment/resources/member_cleanup.go index aba0af2d7..6e531b9ae 100644 --- a/pkg/deployment/resources/member_cleanup.go +++ b/pkg/deployment/resources/member_cleanup.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -49,12 +50,12 @@ var ( ) // CleanupRemovedMembers removes all arangod members that are no longer part of ArangoDB deployment. -func (r *Resources) CleanupRemovedMembers() error { +func (r *Resources) CleanupRemovedMembers(ctx context.Context) error { // Decide what to do depending on cluster mode switch r.context.GetSpec().GetMode() { case api.DeploymentModeCluster: deploymentName := r.context.GetAPIObject().GetName() - if err := r.cleanupRemovedClusterMembers(); err != nil { + if err := r.cleanupRemovedClusterMembers(ctx); err != nil { cleanupRemovedMembersCounters.WithLabelValues(deploymentName, metrics.Failed).Inc() return errors.WithStack(err) } @@ -67,7 +68,7 @@ func (r *Resources) CleanupRemovedMembers() error { } // cleanupRemovedClusterMembers removes all arangod members that are no longer part of the cluster. -func (r *Resources) cleanupRemovedClusterMembers() error { +func (r *Resources) cleanupRemovedClusterMembers(ctx context.Context) error { log := r.log // Fetch recent cluster health @@ -141,7 +142,7 @@ func (r *Resources) cleanupRemovedClusterMembers() error { if updateStatusNeeded { log.Debug().Msg("UpdateStatus needed") - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Warn().Err(err).Msg("Failed to update deployment status") return errors.WithStack(err) } @@ -149,14 +150,14 @@ func (r *Resources) cleanupRemovedClusterMembers() error { for _, podName := range podNamesToRemove { log.Info().Str("pod", podName).Msg("Removing obsolete member pod") - if err := r.context.DeletePod(podName); err != nil && !k8sutil.IsNotFound(err) { + if err := r.context.DeletePod(ctx, podName); err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Str("pod", podName).Msg("Failed to remove obsolete pod") } } for _, pvcName := range pvcNamesToRemove { log.Info().Str("pvc", pvcName).Msg("Removing obsolete member PVC") - if err := r.context.DeletePvc(pvcName); err != nil && !k8sutil.IsNotFound(err) { + if err := r.context.DeletePvc(ctx, pvcName); err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Str("pvc", pvcName).Msg("Failed to remove obsolete PVC") } } @@ -164,7 +165,7 @@ func (r *Resources) cleanupRemovedClusterMembers() error { return nil } -func (r *Resources) EnsureArangoMembers(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureArangoMembers(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { // Create all missing arangomembers s, _ := r.context.GetStatus() @@ -187,7 +188,10 @@ func (r *Resources) EnsureArangoMembers(cachedStatus inspectorInterface.Inspecto }, } - if _, err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Create(context.Background(), &a, metav1.CreateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Create(ctxChild, &a, metav1.CreateOptions{}) + cancel() + if err != nil { return err } @@ -205,7 +209,11 @@ func (r *Resources) EnsureArangoMembers(cachedStatus inspectorInterface.Inspecto if !ok || g != member.Spec.Group { // Remove member - if err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Delete(context.Background(), member.GetName(), metav1.DeleteOptions{}); err != nil { + + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Delete(ctxChild, member.GetName(), metav1.DeleteOptions{}) + cancel() + if err != nil { if !k8sutil.IsNotFound(err) { return err } diff --git a/pkg/deployment/resources/pdbs.go b/pkg/deployment/resources/pdbs.go index a43a14679..0c07a1fda 100644 --- a/pkg/deployment/resources/pdbs.go +++ b/pkg/deployment/resources/pdbs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package resources @@ -44,7 +45,7 @@ func min(a int, b int) int { } // EnsurePDBs ensures Pod Disruption Budgets for different server groups in Cluster mode -func (r *Resources) EnsurePDBs() error { +func (r *Resources) EnsurePDBs(ctx context.Context) error { // Only in Cluster and Production Mode spec := r.context.GetSpec() @@ -65,19 +66,19 @@ func (r *Resources) EnsurePDBs() error { } // Ensure all PDBs as calculated - if err := r.ensurePDBForGroup(api.ServerGroupAgents, minAgents); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupAgents, minAgents); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupDBServers, minDBServers); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupDBServers, minDBServers); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupCoordinators, minCoordinators); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupCoordinators, minCoordinators); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupSyncMasters, minSyncMaster); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupSyncMasters, minSyncMaster); err != nil { return err } - if err := r.ensurePDBForGroup(api.ServerGroupSyncWorkers, minSyncWorker); err != nil { + if err := r.ensurePDBForGroup(ctx, api.ServerGroupSyncWorkers, minSyncWorker); err != nil { return err } } @@ -105,23 +106,25 @@ func newPDB(minAvail int, deplname string, group api.ServerGroup, owner metav1.O } // ensurePDBForGroup ensure pdb for a specific server group, if wantMinAvail is zero, the PDB is removed and not recreated -func (r *Resources) ensurePDBForGroup(group api.ServerGroup, wantedMinAvail int) error { +func (r *Resources) ensurePDBForGroup(ctx context.Context, group api.ServerGroup, wantedMinAvail int) error { deplname := r.context.GetAPIObject().GetName() pdbname := PDBNameForGroup(deplname, group) pdbcli := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()) log := r.log.With().Str("group", group.AsRole()).Logger() - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - for { - pdb, err := pdbcli.Get(context.Background(), pdbname, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + pdb, err := pdbcli.Get(ctxChild, pdbname, metav1.GetOptions{}) + cancel() if k8sutil.IsNotFound(err) { if wantedMinAvail != 0 { // No PDB found - create new pdb := newPDB(wantedMinAvail, deplname, group, r.context.GetAPIObject().AsOwner()) log.Debug().Msg("Creating new PDB") - if _, err := pdbcli.Create(context.Background(), pdb, metav1.CreateOptions{}); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := pdbcli.Create(ctxChild, pdb, metav1.CreateOptions{}) + cancel() + if err != nil { log.Error().Err(err).Msg("failed to create PDB") return errors.WithStack(err) } @@ -142,7 +145,10 @@ func (r *Resources) ensurePDBForGroup(group api.ServerGroup, wantedMinAvail int) // Trigger deletion only if not already deleted if pdb.GetDeletionTimestamp() == nil { // Update the PDB - if err := pdbcli.Delete(context.Background(), pdbname, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := pdbcli.Delete(ctxChild, pdbname, metav1.DeleteOptions{}) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { log.Error().Err(err).Msg("PDB deletion failed") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_cleanup.go b/pkg/deployment/resources/pod_cleanup.go index 501ab5503..d47c31e9b 100644 --- a/pkg/deployment/resources/pod_cleanup.go +++ b/pkg/deployment/resources/pod_cleanup.go @@ -23,6 +23,7 @@ package resources import ( + "context" "time" "github.com/arangodb/kube-arangodb/pkg/deployment/resources/inspector" @@ -42,7 +43,7 @@ const ( // CleanupTerminatedPods removes all pods in Terminated state that belong to a member in Created state. // Returns: Interval_till_next_inspection, error -func (r *Resources) CleanupTerminatedPods(cachedStatus inspectorInterface.Inspector) (util.Interval, error) { +func (r *Resources) CleanupTerminatedPods(ctx context.Context, cachedStatus inspectorInterface.Inspector) (util.Interval, error) { log := r.log nextInterval := maxPodInspectorInterval // Large by default, will be made smaller if needed in the rest of the function @@ -84,7 +85,7 @@ func (r *Resources) CleanupTerminatedPods(cachedStatus inspectorInterface.Inspec // Ok, we can delete the pod log.Debug().Str("pod-name", pod.GetName()).Msg("Cleanup terminated pod") - if err := r.context.CleanupPod(pod); err != nil { + if err := r.context.CleanupPod(ctx, pod); err != nil { log.Warn().Err(err).Str("pod-name", pod.GetName()).Msg("Failed to cleanup pod") } diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index aa46125f6..c73332bc5 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,13 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" "crypto/sha1" "crypto/sha256" "encoding/json" @@ -295,7 +297,7 @@ func (r *Resources) CreatePodTolerations(group api.ServerGroup, groupSpec api.Se return tolerations } -func (r *Resources) RenderPodForMember(cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { +func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspectorInterface.Inspector, spec api.DeploymentSpec, status api.DeploymentStatus, memberID string, imageInfo api.ImageInfo) (*core.Pod, error) { log := r.log apiObject := r.context.GetAPIObject() m, group, found := status.Members.ElementByID(memberID) @@ -365,14 +367,20 @@ func (r *Resources) RenderPodForMember(cachedStatus inspectorInterface.Inspector var tlsKeyfileSecretName, clientAuthCASecretName, masterJWTSecretName, clusterJWTSecretName string // Check master JWT secret - masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, masterJWTSecretName); err != nil { + masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := k8sutil.ValidateTokenSecret(ctxChild, secrets, masterJWTSecretName) + cancel() + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Master JWT secret validation failed")) } monitoringTokenSecretName := spec.Sync.Monitoring.GetTokenSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, monitoringTokenSecretName); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err = k8sutil.ValidateTokenSecret(ctxChild, secrets, monitoringTokenSecretName) + cancel() + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Monitoring token secret validation failed")) } @@ -382,13 +390,20 @@ func (r *Resources) RenderPodForMember(cachedStatus inspectorInterface.Inspector // Check cluster JWT secret if spec.IsAuthenticated() { clusterJWTSecretName = spec.Authentication.GetJWTSecretName() - if err := k8sutil.ValidateTokenSecret(secrets, clusterJWTSecretName); err != nil { + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err = k8sutil.ValidateTokenSecret(ctxChild, secrets, clusterJWTSecretName) + cancel() + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Cluster JWT secret validation failed")) } } // Check client-auth CA certificate secret clientAuthCASecretName = spec.Sync.Authentication.GetClientCASecretName() - if err := k8sutil.ValidateCACertificateSecret(secrets, clientAuthCASecretName); err != nil { + + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err = k8sutil.ValidateCACertificateSecret(ctxChild, secrets, clientAuthCASecretName) + cancel() + if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Client authentication CA certificate secret validation failed")) } } @@ -434,7 +449,7 @@ func (r *Resources) SelectImage(spec api.DeploymentSpec, status api.DeploymentSt } // createPodForMember creates all Pods listed in member status -func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once, cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentSpec, memberID string, imageNotFoundOnce *sync.Once, cachedStatus inspectorInterface.Inspector) error { log := r.log status, lastVersion := r.context.GetStatus() @@ -462,7 +477,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, imageInfo = *m.Image - pod, err := r.RenderPodForMember(cachedStatus, spec, status, memberID, imageInfo) + pod, err := r.RenderPodForMember(ctx, cachedStatus, spec, status, memberID, imageInfo) if err != nil { return errors.WithStack(err) } @@ -495,7 +510,9 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, return errors.WithStack(err) } - uid, err := CreateArangoPod(kubecli, apiObject, spec, group, pod) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) + cancel() if err != nil { return errors.WithStack(err) } @@ -535,7 +552,8 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, } } owner := apiObject.AsOwner() - if err := createTLSServerCertificate(log, secrets, serverNames, spec.Sync.TLS, tlsKeyfileSecretName, &owner); err != nil && !k8sutil.IsAlreadyExists(err) { + err := createTLSServerCertificate(ctx, log, secrets, serverNames, spec.Sync.TLS, tlsKeyfileSecretName, &owner) + if err != nil && !k8sutil.IsAlreadyExists(err) { return errors.WithStack(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } } @@ -545,7 +563,9 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, return errors.WithStack(err) } - uid, err := CreateArangoPod(kubecli, apiObject, spec, group, pod) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) + cancel() if err != nil { return errors.WithStack(err) } @@ -567,7 +587,7 @@ func (r *Resources) createPodForMember(spec api.DeploymentSpec, memberID string, if err := status.Members.Update(m, group); err != nil { return errors.WithStack(err) } - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } // Create event @@ -633,8 +653,8 @@ func RenderArangoPod(deployment k8sutil.APIObject, role, id, podName string, // CreateArangoPod creates a new Pod with container provided by parameter 'containerCreator' // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreateArangoPod(kubecli kubernetes.Interface, deployment k8sutil.APIObject, deploymentSpec api.DeploymentSpec, group api.ServerGroup, pod *core.Pod) (types.UID, error) { - uid, err := k8sutil.CreatePod(kubecli, pod, deployment.GetNamespace(), deployment.AsOwner()) +func CreateArangoPod(ctx context.Context, kubecli kubernetes.Interface, deployment k8sutil.APIObject, deploymentSpec api.DeploymentSpec, group api.ServerGroup, pod *core.Pod) (types.UID, error) { + uid, err := k8sutil.CreatePod(ctx, kubecli, pod, deployment.GetNamespace(), deployment.AsOwner()) if err != nil { return "", errors.WithStack(err) } @@ -660,7 +680,7 @@ func ChecksumArangoPod(groupSpec api.ServerGroupSpec, pod *core.Pod) (string, er } // EnsurePods creates all Pods listed in member status -func (r *Resources) EnsurePods(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePods(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { iterator := r.context.GetServerGroupIterator() deploymentStatus, _ := r.context.GetStatus() imageNotFoundOnce := &sync.Once{} @@ -674,7 +694,7 @@ func (r *Resources) EnsurePods(cachedStatus inspectorInterface.Inspector) error continue } spec := r.context.GetSpec() - if err := r.createPodForMember(spec, m.ID, imageNotFoundOnce, cachedStatus); err != nil { + if err := r.createPodForMember(ctx, spec, m.ID, imageNotFoundOnce, cachedStatus); err != nil { return errors.WithStack(err) } } diff --git a/pkg/deployment/resources/pod_finalizers.go b/pkg/deployment/resources/pod_finalizers.go index 6e05e6061..0ab655e43 100644 --- a/pkg/deployment/resources/pod_finalizers.go +++ b/pkg/deployment/resources/pod_finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -88,8 +89,7 @@ func (r *Resources) runPodFinalizers(ctx context.Context, p *v1.Pod, memberStatu // Remove finalizers (if needed) if len(removalList) > 0 { kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePodFinalizers(log, kubecli, p, removalList, ignoreNotFound); err != nil { + if err := k8sutil.RemovePodFinalizers(ctx, log, kubecli, p, removalList, false); err != nil { log.Debug().Err(err).Msg("Failed to update pod (to remove finalizers)") return 0, errors.WithStack(err) } @@ -119,7 +119,10 @@ func (r *Resources) inspectFinalizerPodAgencyServing(ctx context.Context, log ze // of the agent, also remove the PVC if memberStatus.Conditions.IsTrue(api.ConditionTypeAgentRecoveryNeeded) { pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace()) - if err := pvcs.Delete(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete PVC for member") return errors.WithStack(err) } @@ -146,7 +149,10 @@ func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, log ze // If this DBServer is cleaned out, we need to remove the PVC. if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) || memberStatus.Phase == api.MemberPhaseDrain { pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace()) - if err := pvcs.Delete(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete PVC for member") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_inspector.go b/pkg/deployment/resources/pod_inspector.go index f0abcbe7d..2d2351d08 100644 --- a/pkg/deployment/resources/pod_inspector.go +++ b/pkg/deployment/resources/pod_inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -85,8 +86,8 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter // Remove all finalizers, so it can be removed. log.Warn().Msg("Pod belongs to this deployment, but we don't know the member. Removing all finalizers") kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePodFinalizers(log, kubecli, pod, pod.GetFinalizers(), ignoreNotFound); err != nil { + err := k8sutil.RemovePodFinalizers(ctx, log, kubecli, pod, pod.GetFinalizers(), false) + if err != nil { log.Debug().Err(err).Msg("Failed to update pod (to remove all finalizers)") return errors.WithStack(err) } @@ -322,7 +323,7 @@ func (r *Resources) InspectPods(ctx context.Context, cachedStatus inspectorInter } // Save status - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return 0, errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_termination.go b/pkg/deployment/resources/pod_termination.go index 9c624d19b..081cc5f1e 100644 --- a/pkg/deployment/resources/pod_termination.go +++ b/pkg/deployment/resources/pod_termination.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -58,7 +59,9 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Check node the pod is scheduled on. Only if not in namespaced scope agentDataWillBeGone := false if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" { - node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(context.Background(), p.Spec.NodeName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(ctxChild, p.Spec.NodeName, metav1.GetOptions{}) + cancel() if k8sutil.IsNotFound(err) { log.Warn().Msg("Node not found") } else if err != nil { @@ -71,7 +74,9 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Check PVC pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) - pvc, err := pvcs.Get(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + pvc, err := pvcs.Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + cancel() if err != nil { log.Warn().Err(err).Msg("Failed to get PVC for member") return errors.WithStack(err) @@ -88,10 +93,11 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Inspect agency state log.Debug().Msg("Agent data will be gone, so we will check agency serving status first") - ctx = agency.WithAllowNoLeader(ctx) // The ID we're checking may be the leader, so ignore situations where all other agents are followers - ctx, cancel := context.WithTimeout(ctx, time.Second*15) // Force a quick check + ctxChild, cancel = context.WithTimeout(ctx, time.Second*15) defer cancel() - agencyConns, err := r.context.GetAgencyClients(ctx, func(id string) bool { return id != memberStatus.ID }) + ctxLeader := agency.WithAllowNoLeader(ctxChild) // The ID we're checking may be the leader, so ignore situations where all other agents are followers + + agencyConns, err := r.context.GetAgencyClients(ctxLeader, func(id string) bool { return id != memberStatus.ID }) if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return errors.WithStack(err) @@ -100,7 +106,7 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog log.Debug().Err(err).Msg("No more remaining agents, we cannot delete this one") return errors.WithStack(errors.Newf("No more remaining agents")) } - if err := agency.AreAgentsHealthy(ctx, agencyConns); err != nil { + if err := agency.AreAgentsHealthy(ctxLeader, agencyConns); err != nil { log.Debug().Err(err).Msg("Remaining agents are not healthy") return errors.WithStack(err) } @@ -152,7 +158,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol // Check node the pod is scheduled on dbserverDataWillBeGone := false if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" { - node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(context.Background(), p.Spec.NodeName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(ctxChild, p.Spec.NodeName, metav1.GetOptions{}) + cancel() if k8sutil.IsNotFound(err) { log.Warn().Msg("Node not found") } else if err != nil { @@ -167,7 +175,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol // Check PVC pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) - pvc, err := pvcs.Get(context.Background(), memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + pvc, err := pvcs.Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) + cancel() if err != nil { log.Warn().Err(err).Msg("Failed to get PVC for member") return errors.WithStack(err) @@ -188,12 +198,16 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } // Inspect cleaned out state - c, err := r.context.GetDatabaseClient(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + c, err := r.context.GetDatabaseClient(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return errors.WithStack(err) } - cluster, err := c.Cluster(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cluster, err := c.Cluster(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") @@ -206,7 +220,9 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } return errors.WithStack(err) } - cleanedOut, err := cluster.IsCleanedOut(ctx, memberStatus.ID) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + cleanedOut, err := cluster.IsCleanedOut(ctxChild, memberStatus.ID) + cancel() if err != nil { return errors.WithStack(err) } @@ -244,18 +260,21 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol if memberStatus.Phase == api.MemberPhaseCreated { // No cleanout job triggered var jobID string - ctx = driver.WithJobIDResponse(ctx, &jobID) + ctxChild, cancelChild := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancelChild() + + ctxJobID := driver.WithJobIDResponse(ctxChild, &jobID) // Ensure the cleanout is triggered if dbserverDataWillBeGone { log.Debug().Msg("Server is not yet cleaned out. Triggering a clean out now") - if err := cluster.CleanOutServer(ctx, memberStatus.ID); err != nil { + if err := cluster.CleanOutServer(ctxJobID, memberStatus.ID); err != nil { log.Debug().Err(err).Msg("Failed to clean out server") return errors.WithStack(err) } memberStatus.Phase = api.MemberPhaseDrain } else { log.Debug().Msg("Temporary shutdown, resign leadership") - if err := cluster.ResignServer(ctx, memberStatus.ID); err != nil { + if err := cluster.ResignServer(ctxJobID, memberStatus.ID); err != nil { log.Debug().Err(err).Msg("Failed to resign server") return errors.WithStack(err) } @@ -269,12 +288,16 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } } else if memberStatus.Phase == api.MemberPhaseDrain { // Check the job progress - agency, err := r.context.GetAgency(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + agency, err := r.context.GetAgency(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, memberStatus.CleanoutJobID, c, agency) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch job status") return errors.WithStack(err) @@ -296,12 +319,17 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } } else if memberStatus.Phase == api.MemberPhaseResign { // Check the job progress - agency, err := r.context.GetAgency(ctx) + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + agency, err := r.context.GetAgency(ctxChild) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return errors.WithStack(err) } - jobStatus, err := arangod.CleanoutServerJobStatus(ctx, memberStatus.CleanoutJobID, c, agency) + + ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch job status") return errors.WithStack(err) diff --git a/pkg/deployment/resources/pvc_finalizers.go b/pkg/deployment/resources/pvc_finalizers.go index e3cdd3d66..011cf47c2 100644 --- a/pkg/deployment/resources/pvc_finalizers.go +++ b/pkg/deployment/resources/pvc_finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -43,14 +44,15 @@ const ( ) // runPVCFinalizers goes through the list of PVC finalizers to see if they can be removed. -func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) (util.Interval, error) { +func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolumeClaim, group api.ServerGroup, + memberStatus api.MemberStatus) (util.Interval, error) { log := r.log.With().Str("pvc-name", p.GetName()).Logger() var removalList []string for _, f := range p.ObjectMeta.GetFinalizers() { switch f { case constants.FinalizerPVCMemberExists: log.Debug().Msg("Inspecting member exists finalizer") - if err := r.inspectFinalizerPVCMemberExists(ctx, log, p, group, memberStatus); err == nil { + if err := r.inspectFinalizerPVCMemberExists(ctx, log, group, memberStatus); err == nil { removalList = append(removalList, f) } else { log.Debug().Err(err).Str("finalizer", f).Msg("Cannot remove finalizer yet") @@ -60,8 +62,8 @@ func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolume // Remove finalizers (if needed) if len(removalList) > 0 { kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePVCFinalizers(log, kubecli, p, removalList, ignoreNotFound); err != nil { + err := k8sutil.RemovePVCFinalizers(ctx, log, kubecli, p, removalList, false) + if err != nil { log.Debug().Err(err).Msg("Failed to update PVC (to remove finalizers)") return 0, errors.WithStack(err) } @@ -74,7 +76,8 @@ func (r *Resources) runPVCFinalizers(ctx context.Context, p *v1.PersistentVolume // inspectFinalizerPVCMemberExists checks the finalizer condition for member-exists. // It returns nil if the finalizer can be removed. -func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zerolog.Logger, p *v1.PersistentVolumeClaim, group api.ServerGroup, memberStatus api.MemberStatus) error { +func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zerolog.Logger, group api.ServerGroup, + memberStatus api.MemberStatus) error { // Inspect member phase if memberStatus.Phase.IsFailed() { log.Debug().Msg("Member is already failed, safe to remove member-exists finalizer") @@ -105,7 +108,10 @@ func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zer if memberStatus.PodName != "" { log.Info().Msg("Removing Pod of member, because PVC is being removed") pods := r.context.GetKubeCli().CoreV1().Pods(apiObject.GetNamespace()) - if err := pods.Delete(context.Background(), memberStatus.PodName, metav1.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := pods.Delete(ctxChild, memberStatus.PodName, metav1.DeleteOptions{}) + cancel() + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Msg("Failed to delete pod") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pvc_inspector.go b/pkg/deployment/resources/pvc_inspector.go index 2319f23fc..338f7c6e1 100644 --- a/pkg/deployment/resources/pvc_inspector.go +++ b/pkg/deployment/resources/pvc_inspector.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -70,8 +71,8 @@ func (r *Resources) InspectPVCs(ctx context.Context, cachedStatus inspectorInter // Remove all finalizers, so it can be removed. log.Warn().Msg("PVC belongs to this deployment, but we don't know the member. Removing all finalizers") kubecli := r.context.GetKubeCli() - ignoreNotFound := false - if err := k8sutil.RemovePVCFinalizers(log, kubecli, pvc, pvc.GetFinalizers(), ignoreNotFound); err != nil { + err := k8sutil.RemovePVCFinalizers(ctx, log, kubecli, pvc, pvc.GetFinalizers(), false) + if err != nil { log.Debug().Err(err).Msg("Failed to update PVC (to remove all finalizers)") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pvcs.go b/pkg/deployment/resources/pvcs.go index f8679f5a5..4876d9d44 100644 --- a/pkg/deployment/resources/pvcs.go +++ b/pkg/deployment/resources/pvcs.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources import ( + "context" + api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1" "github.com/arangodb/kube-arangodb/pkg/util/constants" "github.com/arangodb/kube-arangodb/pkg/util/errors" @@ -36,7 +39,7 @@ func (r *Resources) createPVCFinalizers(group api.ServerGroup) []string { } // EnsurePVCs creates all PVC's listed in member status -func (r *Resources) EnsurePVCs(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsurePVCs(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { kubecli := r.context.GetKubeCli() apiObject := r.context.GetAPIObject() deploymentName := apiObject.GetName() @@ -62,7 +65,10 @@ func (r *Resources) EnsurePVCs(cachedStatus inspectorInterface.Inspector) error resources := spec.Resources vct := spec.VolumeClaimTemplate finalizers := r.createPVCFinalizers(group) - if err := k8sutil.CreatePersistentVolumeClaim(pvcs, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, vct, finalizers, owner); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := k8sutil.CreatePersistentVolumeClaim(ctxChild, pvcs, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, vct, finalizers, owner) + cancel() + if err != nil { return errors.WithStack(err) } } diff --git a/pkg/deployment/resources/secret_hashes.go b/pkg/deployment/resources/secret_hashes.go index 36d0e99cd..8d4f602b0 100644 --- a/pkg/deployment/resources/secret_hashes.go +++ b/pkg/deployment/resources/secret_hashes.go @@ -50,7 +50,7 @@ import ( // If a hash is different, the deployment is marked // with a SecretChangedCondition and the operator will not // touch it until this is resolved. -func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) ValidateSecretHashes(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { // validate performs a secret hash comparison for a single secret. // Return true if all is good, false when the SecretChanged condition // must be set. @@ -130,7 +130,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspect status.SecretHashes.Users = make(map[string]string) } updater(status.SecretHashes) - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } // Reload status @@ -210,7 +210,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspect if status.Conditions.Update(api.ConditionTypeSecretsChanged, true, "Secrets have changed", fmt.Sprintf("Found %d changed secrets", len(badSecretNames))) { log.Warn().Msgf("Found %d changed secrets. Settings SecretsChanged condition", len(badSecretNames)) - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Error().Err(err).Msg("Failed to save SecretsChanged condition") return errors.WithStack(err) } @@ -221,7 +221,7 @@ func (r *Resources) ValidateSecretHashes(cachedStatus inspectorInterface.Inspect // All good, we van remove the SecretsChanged condition if status.Conditions.Remove(api.ConditionTypeSecretsChanged) { log.Info().Msg("Resetting SecretsChanged condition") - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { log.Error().Err(err).Msg("Failed to save SecretsChanged condition") return errors.WithStack(err) } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 6ff77d19c..88b2877e0 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -72,7 +73,7 @@ func GetCASecretName(apiObject k8sutil.APIObject) string { } // EnsureSecrets creates all secrets needed to run the given deployment -func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureSecrets(ctx context.Context, log zerolog.Logger, cachedStatus inspectorInterface.Inspector) error { start := time.Now() spec := r.context.GetSpec() kubecli := r.context.GetKubeCli() @@ -88,13 +89,13 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte if spec.IsAuthenticated() { counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTokenSecret(cachedStatus, secrets, spec.Authentication.GetJWTSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTokenSecret(ctx, cachedStatus, secrets, spec.Authentication.GetJWTSecretName())); err != nil { return errors.WithStack(err) } if imageFound { if pod.VersionHasJWTSecretKeyfolder(image.ArangoDBVersion, image.Enterprise) { - if err := r.ensureTokenSecretFolder(cachedStatus, secrets, spec.Authentication.GetJWTSecretName(), pod.JWTSecretFolder(deploymentName)); err != nil { + if err := r.ensureTokenSecretFolder(ctx, cachedStatus, secrets, spec.Authentication.GetJWTSecretName(), pod.JWTSecretFolder(deploymentName)); err != nil { return errors.WithStack(err) } } @@ -102,11 +103,11 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte if spec.Metrics.IsEnabled() { if imageFound && pod.VersionHasJWTSecretKeyfolder(image.ArangoDBVersion, image.Enterprise) { - if err := r.refreshCache(cachedStatus, r.ensureExporterTokenSecret(cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), pod.JWTSecretFolder(deploymentName))); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureExporterTokenSecret(ctx, cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), pod.JWTSecretFolder(deploymentName))); err != nil { return errors.WithStack(err) } } else { - if err := r.refreshCache(cachedStatus, r.ensureExporterTokenSecret(cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureExporterTokenSecret(ctx, cachedStatus, secrets, spec.Metrics.GetJWTTokenSecretName(), spec.Authentication.GetJWTSecretName())); err != nil { return errors.WithStack(err) } } @@ -114,11 +115,11 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte } if spec.IsSecure() { counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTLSCACertificateSecret(cachedStatus, secrets, spec.TLS)); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTLSCACertificateSecret(ctx, cachedStatus, secrets, spec.TLS)); err != nil { return errors.WithStack(err) } - if err := r.refreshCache(cachedStatus, r.ensureSecretWithEmptyKey(cachedStatus, secrets, GetCASecretName(r.context.GetAPIObject()), "empty")); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureSecretWithEmptyKey(ctx, cachedStatus, secrets, GetCASecretName(r.context.GetAPIObject()), "empty")); err != nil { return errors.WithStack(err) } @@ -162,11 +163,12 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte serverNames = append(serverNames, ip) } owner := member.AsOwner() - if err := r.refreshCache(cachedStatus, createTLSServerCertificate(log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner)); err != nil && !k8sutil.IsAlreadyExists(err) { + errCert := createTLSServerCertificate(ctx, log, secrets, serverNames, spec.TLS, tlsKeyfileSecretName, &owner) + if err := r.refreshCache(ctx, cachedStatus, errCert); err != nil && !k8sutil.IsAlreadyExists(err) { return errors.WithStack(errors.Wrapf(err, "Failed to create TLS keyfile secret")) } - if err := r.refreshCache(cachedStatus, operatorErrors.Reconcile()); err != nil { + if err := r.refreshCache(ctx, cachedStatus, operatorErrors.Reconcile()); err != nil { return errors.WithStack(err) } } @@ -178,39 +180,40 @@ func (r *Resources) EnsureSecrets(log zerolog.Logger, cachedStatus inspectorInte } if spec.RocksDB.IsEncrypted() { if i := status.CurrentImage; i != nil && features.EncryptionRotation().Supported(i.ArangoDBVersion, i.Enterprise) { - if err := r.refreshCache(cachedStatus, r.ensureEncryptionKeyfolderSecret(cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetEncryptionFolderSecretName(deploymentName))); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureEncryptionKeyfolderSecret(ctx, cachedStatus, secrets, spec.RocksDB.Encryption.GetKeySecretName(), pod.GetEncryptionFolderSecretName(deploymentName))); err != nil { return errors.WithStack(err) } } } if spec.Sync.IsEnabled() { counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTokenSecret(cachedStatus, secrets, spec.Sync.Authentication.GetJWTSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTokenSecret(ctx, cachedStatus, secrets, spec.Sync.Authentication.GetJWTSecretName())); err != nil { return errors.WithStack(err) } counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTokenSecret(cachedStatus, secrets, spec.Sync.Monitoring.GetTokenSecretName())); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTokenSecret(ctx, cachedStatus, secrets, spec.Sync.Monitoring.GetTokenSecretName())); err != nil { return errors.WithStack(err) } counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureTLSCACertificateSecret(cachedStatus, secrets, spec.Sync.TLS)); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureTLSCACertificateSecret(ctx, cachedStatus, secrets, spec.Sync.TLS)); err != nil { return errors.WithStack(err) } counterMetric.Inc() - if err := r.refreshCache(cachedStatus, r.ensureClientAuthCACertificateSecret(cachedStatus, secrets, spec.Sync.Authentication)); err != nil { + if err := r.refreshCache(ctx, cachedStatus, r.ensureClientAuthCACertificateSecret(ctx, cachedStatus, secrets, spec.Sync.Authentication)); err != nil { return errors.WithStack(err) } } return nil } -func (r *Resources) refreshCache(cachedStatus inspectorInterface.Inspector, err error) error { +func (r *Resources) refreshCache(ctx context.Context, cachedStatus inspectorInterface.Inspector, err error) error { if err == nil { return nil } if operatorErrors.IsReconcile(err) { - if err := cachedStatus.Refresh(r.context.GetKubeCli(), r.context.GetMonitoringV1Cli(), r.context.GetArangoCli(), r.context.GetNamespace()); err != nil { + err := cachedStatus.Refresh(ctx, r.context.GetKubeCli(), r.context.GetMonitoringV1Cli(), r.context.GetArangoCli(), r.context.GetNamespace()) + if err != nil { return errors.WithStack(err) } } else { @@ -220,7 +223,7 @@ func (r *Resources) refreshCache(cachedStatus inspectorInterface.Inspector, err return nil } -func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, folderSecretName string) error { +func (r *Resources) ensureTokenSecretFolder(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, folderSecretName string) error { if f, exists := cachedStatus.Secret(folderSecretName); exists { if len(f.Data) == 0 { s, exists := cachedStatus.Secret(secretName) @@ -237,7 +240,10 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp f.Data[pod.ActiveJWTKey] = token f.Data[constants.SecretKeyToken] = token - if _, err := secrets.Update(context.Background(), f, meta.UpdateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := secrets.Update(ctxChild, f, meta.UpdateOptions{}) + cancel() + if err != nil { return err } @@ -258,7 +264,11 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return err } - if _, err := secrets.Patch(context.Background(), folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err = secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) + cancel() + + if err != nil { return err } } @@ -277,7 +287,10 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return err } - if _, err := secrets.Patch(context.Background(), folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err = secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) + cancel() + if err != nil { return err } } @@ -295,7 +308,7 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return errors.Newf("Token secret is invalid") } - if err := r.createSecretWithMod(secrets, folderSecretName, func(s *core.Secret) { + if err := r.createSecretWithMod(ctx, secrets, folderSecretName, func(s *core.Secret) { s.Data[util.SHA256(token)] = token s.Data[pod.ActiveJWTKey] = token s.Data[constants.SecretKeyToken] = token @@ -306,9 +319,9 @@ func (r *Resources) ensureTokenSecretFolder(cachedStatus inspectorInterface.Insp return nil } -func (r *Resources) ensureTokenSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName string) error { +func (r *Resources) ensureTokenSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName string) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createTokenSecret(secrets, secretName) + return r.createTokenSecret(ctx, secrets, secretName) } return nil @@ -340,23 +353,23 @@ func (r *Resources) createSecret(secrets k8sutil.SecretInterface, secretName str return operatorErrors.Reconcile() } -func (r *Resources) ensureSecretWithEmptyKey(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error { +func (r *Resources) ensureSecretWithEmptyKey(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createSecretWithKey(secrets, secretName, keyName, nil) + return r.createSecretWithKey(ctx, secrets, secretName, keyName, nil) } return nil } -func (r *Resources) ensureSecretWithKey(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { +func (r *Resources) ensureSecretWithKey(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { if _, exists := cachedStatus.Secret(secretName); !exists { - return r.createSecretWithKey(secrets, secretName, keyName, value) + return r.createSecretWithKey(ctx, secrets, secretName, keyName, value) } return nil } -func (r *Resources) createSecretWithMod(secrets k8sutil.SecretInterface, secretName string, f func(s *core.Secret)) error { +func (r *Resources) createSecretWithMod(ctx context.Context, secrets k8sutil.SecretInterface, secretName string, f func(s *core.Secret)) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -370,7 +383,10 @@ func (r *Resources) createSecretWithMod(secrets k8sutil.SecretInterface, secretN f(secret) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}) + cancel() + if err != nil { // Failed to create secret return errors.WithStack(err) } @@ -378,20 +394,23 @@ func (r *Resources) createSecretWithMod(secrets k8sutil.SecretInterface, secretN return operatorErrors.Reconcile() } -func (r *Resources) createSecretWithKey(secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { - return r.createSecretWithMod(secrets, secretName, func(s *core.Secret) { +func (r *Resources) createSecretWithKey(ctx context.Context, secrets k8sutil.SecretInterface, secretName, keyName string, value []byte) error { + return r.createSecretWithMod(ctx, secrets, secretName, func(s *core.Secret) { s.Data[keyName] = value }) } -func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretName string) error { +func (r *Resources) createTokenSecret(ctx context.Context, secrets k8sutil.SecretInterface, secretName string) error { tokenData := make([]byte, 32) rand.Read(tokenData) token := hex.EncodeToString(tokenData) // Create secret owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateTokenSecret(secrets, secretName, token, &owner); k8sutil.IsAlreadyExists(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := k8sutil.CreateTokenSecret(ctxChild, secrets, secretName, token, &owner) + cancel() + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -402,7 +421,7 @@ func (r *Resources) createTokenSecret(secrets k8sutil.SecretInterface, secretNam return operatorErrors.Reconcile() } -func (r *Resources) ensureEncryptionKeyfolderSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error { +func (r *Resources) ensureEncryptionKeyfolderSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, keyfileSecretName, secretName string) error { _, folderExists := cachedStatus.Secret(secretName) keyfile, exists := cachedStatus.Secret(keyfileSecretName) @@ -429,13 +448,17 @@ func (r *Resources) ensureEncryptionKeyfolderSecret(cachedStatus inspectorInterf } owner := r.context.GetAPIObject().AsOwner() - if err := AppendKeyfileToKeyfolder(cachedStatus, secrets, &owner, secretName, d); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := AppendKeyfileToKeyfolder(ctxChild, cachedStatus, secrets, &owner, secretName, d) + cancel() + if err != nil { return errors.Wrapf(err, "Unable to create keyfolder secret") } return nil } -func AppendKeyfileToKeyfolder(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, ownerRef *meta.OwnerReference, secretName string, encryptionKey []byte) error { +func AppendKeyfileToKeyfolder(ctx context.Context, cachedStatus inspectorInterface.Inspector, + secrets k8sutil.SecretInterface, ownerRef *meta.OwnerReference, secretName string, encryptionKey []byte) error { encSha := fmt.Sprintf("%0x", sha256.Sum256(encryptionKey)) if _, exists := cachedStatus.Secret(secretName); !exists { @@ -450,7 +473,7 @@ func AppendKeyfileToKeyfolder(cachedStatus inspectorInterface.Inspector, secrets } // Attach secret to owner k8sutil.AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -471,14 +494,16 @@ var ( // ensureExporterTokenSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with correct access. -func (r *Resources) ensureExporterTokenSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) error { +func (r *Resources) ensureExporterTokenSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, + secrets k8sutil.SecretInterface, tokenSecretName, secretSecretName string) error { if update, exists, err := r.ensureExporterTokenSecretCreateRequired(cachedStatus, tokenSecretName, secretSecretName); err != nil { return err } else if update { // Create secret if !exists { owner := r.context.GetAPIObject().AsOwner() - if err := k8sutil.CreateJWTFromSecret(secrets, tokenSecretName, secretSecretName, exporterTokenClaims, &owner); k8sutil.IsAlreadyExists(err) { + err = k8sutil.CreateJWTFromSecret(ctx, secrets, tokenSecretName, secretSecretName, exporterTokenClaims, &owner) + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -531,13 +556,16 @@ func (r *Resources) ensureExporterTokenSecretCreateRequired(cachedStatus inspect // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -func (r *Resources) ensureTLSCACertificateSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec) error { +func (r *Resources) ensureTLSCACertificateSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec) error { if _, exists := cachedStatus.Secret(spec.GetCASecretName()); !exists { // Secret not found, create it apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - if err := createTLSCACertificate(r.log, secrets, spec, deploymentName, &owner); k8sutil.IsAlreadyExists(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := createTLSCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) + cancel() + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -552,7 +580,7 @@ func (r *Resources) ensureTLSCACertificateSecret(cachedStatus inspectorInterface // ensureTLSCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -func (r *Resources) ensureTLSCAFolderSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec, folderSecretName string) error { +func (r *Resources) ensureTLSCAFolderSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.TLSSpec, folderSecretName string) error { if spec.CASecretName == nil { return errors.Newf("CA Secret Name is nil") } @@ -580,7 +608,7 @@ func (r *Resources) ensureTLSCAFolderSecret(cachedStatus inspectorInterface.Insp certSha := util.SHA256(caData) // Secret not found, create it - return r.createSecretWithMod(secrets, folderSecretName, func(s *core.Secret) { + return r.createSecretWithMod(ctx, secrets, folderSecretName, func(s *core.Secret) { s.Data[certSha] = caData }) } @@ -589,13 +617,16 @@ func (r *Resources) ensureTLSCAFolderSecret(cachedStatus inspectorInterface.Insp // ensureClientAuthCACertificateSecret checks if a secret with given name exists in the namespace // of the deployment. If not, it will add such a secret with a generated CA certificate. -func (r *Resources) ensureClientAuthCACertificateSecret(cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec) error { +func (r *Resources) ensureClientAuthCACertificateSecret(ctx context.Context, cachedStatus inspectorInterface.Inspector, secrets k8sutil.SecretInterface, spec api.SyncAuthenticationSpec) error { if _, exists := cachedStatus.Secret(spec.GetClientCASecretName()); !exists { // Secret not found, create it apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - if err := createClientAuthCACertificate(r.log, secrets, spec, deploymentName, &owner); k8sutil.IsAlreadyExists(err) { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := createClientAuthCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) + cancel() + if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil } else if err != nil { @@ -617,7 +648,7 @@ func (r *Resources) getJWTSecret(spec api.DeploymentSpec) (string, error) { ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := spec.Authentication.GetJWTSecretName() - s, err := k8sutil.GetTokenSecret(secrets, secretName) + s, err := k8sutil.GetTokenSecret(context.TODO(), secrets, secretName) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get JWT secret") return "", errors.WithStack(err) @@ -631,7 +662,7 @@ func (r *Resources) getSyncJWTSecret(spec api.DeploymentSpec) (string, error) { ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := spec.Sync.Authentication.GetJWTSecretName() - s, err := k8sutil.GetTokenSecret(secrets, secretName) + s, err := k8sutil.GetTokenSecret(context.TODO(), secrets, secretName) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync JWT secret") return "", errors.WithStack(err) @@ -645,7 +676,7 @@ func (r *Resources) getSyncMonitoringToken(spec api.DeploymentSpec) (string, err ns := r.context.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := spec.Sync.Monitoring.GetTokenSecretName() - s, err := k8sutil.GetTokenSecret(secrets, secretName) + s, err := k8sutil.GetTokenSecret(context.TODO(), secrets, secretName) if err != nil { r.log.Debug().Err(err).Str("secret-name", secretName).Msg("Failed to get sync monitoring secret") return "", errors.WithStack(err) diff --git a/pkg/deployment/resources/servicemonitor.go b/pkg/deployment/resources/servicemonitor.go index 0ec771671..aa9058298 100644 --- a/pkg/deployment/resources/servicemonitor.go +++ b/pkg/deployment/resources/servicemonitor.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2019 ArangoDB Inc, Cologne, Germany +// Copyright 2019-2021 ArangoDB Inc, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Max Neunhoeffer +// Author Tomasz Mielech // package resources @@ -138,7 +139,7 @@ func (r *Resources) serviceMonitorSpec() (coreosv1.ServiceMonitorSpec, error) { } // EnsureServiceMonitor creates or updates a ServiceMonitor. -func (r *Resources) EnsureServiceMonitor() error { +func (r *Resources) EnsureServiceMonitor(ctx context.Context) error { // Some preparations: log := r.log apiObject := r.context.GetAPIObject() @@ -157,7 +158,9 @@ func (r *Resources) EnsureServiceMonitor() error { // Check if ServiceMonitor already exists serviceMonitors := mClient.ServiceMonitors(ns) - servMon, err := serviceMonitors.Get(context.Background(), serviceMonitorName, metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + servMon, err := serviceMonitors.Get(ctxChild, serviceMonitorName, metav1.GetOptions{}) + cancel() if err != nil { if k8sutil.IsNotFound(err) { if !wantMetrics { @@ -178,7 +181,10 @@ func (r *Resources) EnsureServiceMonitor() error { }, Spec: spec, } - smon, err = serviceMonitors.Create(context.Background(), smon, metav1.CreateOptions{}) + + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + smon, err = serviceMonitors.Create(ctxChild, smon, metav1.CreateOptions{}) + cancel() if err != nil { log.Error().Err(err).Msgf("Failed to create ServiceMonitor %s", serviceMonitorName) return errors.WithStack(err) @@ -220,7 +226,9 @@ func (r *Resources) EnsureServiceMonitor() error { servMon.Spec = spec - _, err = serviceMonitors.Update(context.Background(), servMon, metav1.UpdateOptions{}) + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err = serviceMonitors.Update(ctxChild, servMon, metav1.UpdateOptions{}) + cancel() if err != nil { return err } @@ -228,7 +236,9 @@ func (r *Resources) EnsureServiceMonitor() error { return nil } // Need to get rid of the ServiceMonitor: - err = serviceMonitors.Delete(context.Background(), serviceMonitorName, metav1.DeleteOptions{}) + ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err = serviceMonitors.Delete(ctxChild, serviceMonitorName, metav1.DeleteOptions{}) + cancel() if err == nil { log.Debug().Msgf("Deleted ServiceMonitor %s", serviceMonitorName) return nil diff --git a/pkg/deployment/resources/services.go b/pkg/deployment/resources/services.go index c5e33d83f..420ba821c 100644 --- a/pkg/deployment/resources/services.go +++ b/pkg/deployment/resources/services.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package resources @@ -49,7 +50,7 @@ var ( ) // EnsureServices creates all services needed to service the deployment -func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) error { +func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorInterface.Inspector) error { log := r.log start := time.Now() kubecli := r.context.GetKubeCli() @@ -99,7 +100,10 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er }, } - if _, err := svcs.Create(context.Background(), s, metav1.CreateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := svcs.Create(ctxChild, s, metav1.CreateOptions{}) + cancel() + if err != nil { if !k8sutil.IsConflict(err) { return err } @@ -124,7 +128,10 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er if !equality.Semantic.DeepDerivative(*spec, s.Spec) { s.Spec = *spec - if _, err := svcs.Update(context.Background(), s, metav1.UpdateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := svcs.Update(ctxChild, s, metav1.UpdateOptions{}) + cancel() + if err != nil { return err } @@ -141,7 +148,9 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er // Headless service counterMetric.Inc() if _, exists := cachedStatus.Service(k8sutil.CreateHeadlessServiceName(deploymentName)); !exists { - svcName, newlyCreated, err := k8sutil.CreateHeadlessService(svcs, apiObject, owner) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + svcName, newlyCreated, err := k8sutil.CreateHeadlessService(ctxChild, svcs, apiObject, owner) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create headless service") return errors.WithStack(err) @@ -155,7 +164,9 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er single := spec.GetMode().HasSingleServers() counterMetric.Inc() if _, exists := cachedStatus.Service(k8sutil.CreateDatabaseClientServiceName(deploymentName)); !exists { - svcName, newlyCreated, err := k8sutil.CreateDatabaseClientService(svcs, apiObject, single, owner) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + svcName, newlyCreated, err := k8sutil.CreateDatabaseClientService(ctxChild, svcs, apiObject, single, owner) + cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create database client service") return errors.WithStack(err) @@ -167,7 +178,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er status, lastVersion := r.context.GetStatus() if status.ServiceName != svcName { status.ServiceName = svcName - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -180,7 +191,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er if single { role = "single" } - if err := r.ensureExternalAccessServices(cachedStatus, svcs, eaServiceName, ns, role, "database", k8sutil.ArangoPort, false, spec.ExternalAccess, apiObject, log, counterMetric); err != nil { + if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, ns, role, "database", k8sutil.ArangoPort, false, spec.ExternalAccess, apiObject, log, counterMetric); err != nil { return errors.WithStack(err) } @@ -189,20 +200,22 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er counterMetric.Inc() eaServiceName := k8sutil.CreateSyncMasterClientServiceName(deploymentName) role := "syncmaster" - if err := r.ensureExternalAccessServices(cachedStatus, svcs, eaServiceName, ns, role, "sync", k8sutil.ArangoSyncMasterPort, true, spec.Sync.ExternalAccess.ExternalAccessSpec, apiObject, log, counterMetric); err != nil { + if err := r.ensureExternalAccessServices(ctx, cachedStatus, svcs, eaServiceName, ns, role, "sync", k8sutil.ArangoSyncMasterPort, true, spec.Sync.ExternalAccess.ExternalAccessSpec, apiObject, log, counterMetric); err != nil { return errors.WithStack(err) } status, lastVersion := r.context.GetStatus() if status.SyncServiceName != eaServiceName { status.SyncServiceName = eaServiceName - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } } if spec.Metrics.IsEnabled() { - name, _, err := k8sutil.CreateExporterService(cachedStatus, svcs, apiObject, apiObject.AsOwner()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + name, _, err := k8sutil.CreateExporterService(ctxChild, cachedStatus, svcs, apiObject, apiObject.AsOwner()) + cancel() if err != nil { log.Debug().Err(err).Msgf("Failed to create %s exporter service", name) return errors.WithStack(err) @@ -210,7 +223,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er status, lastVersion := r.context.GetStatus() if status.ExporterServiceName != name { status.ExporterServiceName = name - if err := r.context.UpdateStatus(status, lastVersion); err != nil { + if err := r.context.UpdateStatus(ctx, status, lastVersion); err != nil { return errors.WithStack(err) } } @@ -219,7 +232,7 @@ func (r *Resources) EnsureServices(cachedStatus inspectorInterface.Inspector) er } // EnsureServices creates all services needed to service the deployment -func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface.Inspector, svcs k8sutil.ServiceInterface, eaServiceName, ns, svcRole, title string, port int, noneIsClusterIP bool, spec api.ExternalAccessSpec, apiObject k8sutil.APIObject, log zerolog.Logger, counterMetric prometheus.Counter) error { +func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStatus inspectorInterface.Inspector, svcs k8sutil.ServiceInterface, eaServiceName, ns, svcRole, title string, port int, noneIsClusterIP bool, spec api.ExternalAccessSpec, apiObject k8sutil.APIObject, log zerolog.Logger, counterMetric prometheus.Counter) error { // Database external access service createExternalAccessService := false deleteExternalAccessService := false @@ -275,7 +288,10 @@ func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface } } if updateExternalAccessService && !createExternalAccessService && !deleteExternalAccessService { - if _, err := svcs.Update(context.Background(), existing, metav1.UpdateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, err := svcs.Update(ctxChild, existing, metav1.UpdateOptions{}) + cancel() + if err != nil { log.Debug().Err(err).Msgf("Failed to update %s external access service", title) return errors.WithStack(err) } @@ -289,7 +305,10 @@ func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface if deleteExternalAccessService { log.Info().Str("service", eaServiceName).Msgf("Removing obsolete %s external access service", title) - if err := svcs.Delete(context.Background(), eaServiceName, metav1.DeleteOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + err := svcs.Delete(ctxChild, eaServiceName, metav1.DeleteOptions{}) + cancel() + if err != nil { log.Debug().Err(err).Msgf("Failed to remove %s external access service", title) return errors.WithStack(err) } @@ -299,7 +318,9 @@ func (r *Resources) ensureExternalAccessServices(cachedStatus inspectorInterface nodePort := spec.GetNodePort() loadBalancerIP := spec.GetLoadBalancerIP() loadBalancerSourceRanges := spec.LoadBalancerSourceRanges - _, newlyCreated, err := k8sutil.CreateExternalAccessService(svcs, eaServiceName, svcRole, apiObject, eaServiceType, port, nodePort, loadBalancerIP, loadBalancerSourceRanges, apiObject.AsOwner()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + _, newlyCreated, err := k8sutil.CreateExternalAccessService(ctxChild, svcs, eaServiceName, svcRole, apiObject, eaServiceType, port, nodePort, loadBalancerIP, loadBalancerSourceRanges, apiObject.AsOwner()) + cancel() if err != nil { log.Debug().Err(err).Msgf("Failed to create %s external access service", title) return errors.WithStack(err) diff --git a/pkg/replication/server_endpoint_api.go b/pkg/replication/server_endpoint_api.go index 24fb937d1..6242bc7ac 100644 --- a/pkg/replication/server_endpoint_api.go +++ b/pkg/replication/server_endpoint_api.go @@ -23,6 +23,8 @@ package replication import ( + "context" + api "github.com/arangodb/kube-arangodb/pkg/apis/replication/v1" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil" ) @@ -58,7 +60,7 @@ func (ep serverEndpoint) AuthUserSecretName() string { func (ep serverEndpoint) TLSCACert() string { tlsCASecretName := ep.getSpec().TLS.GetCASecretName() secrets := ep.dr.deps.KubeCli.CoreV1().Secrets(ep.dr.apiObject.GetNamespace()) - caCert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + caCert, err := k8sutil.GetCACertficateSecret(context.TODO(), secrets, tlsCASecretName) if err != nil { return "" } diff --git a/pkg/replication/sync_client.go b/pkg/replication/sync_client.go index d5dfc1bf5..fbfa07f1c 100644 --- a/pkg/replication/sync_client.go +++ b/pkg/replication/sync_client.go @@ -67,7 +67,7 @@ func (dr *DeploymentReplication) createSyncMasterClient(epSpec api.EndpointSpec) } } else if authJWTSecretName != "" { var err error - jwtSecret, err = k8sutil.GetTokenSecret(secrets, authJWTSecretName) + jwtSecret, err = k8sutil.GetTokenSecret(context.TODO(), secrets, authJWTSecretName) if err != nil { return nil, errors.WithStack(err) } @@ -86,7 +86,7 @@ func (dr *DeploymentReplication) createSyncMasterClient(epSpec api.EndpointSpec) } } if tlsCASecretName != "" { - caCert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + caCert, err := k8sutil.GetCACertficateSecret(context.TODO(), secrets, tlsCASecretName) if err != nil { return nil, errors.WithStack(err) } @@ -141,7 +141,7 @@ func (dr *DeploymentReplication) createArangoSyncTLSAuthentication(spec api.Depl } // Fetch TLS CA certificate for source - caCert, err := k8sutil.GetCACertficateSecret(secrets, tlsCASecretName) + caCert, err := k8sutil.GetCACertficateSecret(context.TODO(), secrets, tlsCASecretName) if err != nil { return client.TLSAuthentication{}, errors.WithStack(err) } diff --git a/pkg/util/arangod/cleanout_server.go b/pkg/util/arangod/cleanout_server.go index 0111843f0..cf98e7bf0 100644 --- a/pkg/util/arangod/cleanout_server.go +++ b/pkg/util/arangod/cleanout_server.go @@ -28,7 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/util/errors" - driver "github.com/arangodb/go-driver" + "github.com/arangodb/go-driver" "github.com/arangodb/go-driver/agency" ) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index bc6790843..47508db66 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package arangod @@ -49,6 +50,24 @@ type ( requireAuthenticationKey struct{} ) +const ( + minArangoDDefaultTimeout = time.Second * 10 +) + +var requestTimeout = minArangoDDefaultTimeout + +// GetRequestTimeout gets request timeout for one call to kubernetes. +func GetRequestTimeout() time.Duration { + return requestTimeout +} + +// SetRequestTimeout sets request timeout for one call to kubernetes. +func SetRequestTimeout(timeout time.Duration) { + if timeout > minArangoDDefaultTimeout { + requestTimeout = timeout + } +} + // WithSkipAuthentication prepares a context that when given to functions in // this file will avoid creating any authentication for arango clients. func WithSkipAuthentication(ctx context.Context) context.Context { @@ -258,7 +277,9 @@ func createArangodClientAuthentication(ctx context.Context, cli corev1.CoreV1Int // Should we skip using it? if ctx.Value(skipAuthenticationKey{}) == nil { secrets := cli.Secrets(apiObject.GetNamespace()) - s, err := k8sutil.GetTokenSecret(secrets, apiObject.Spec.Authentication.GetJWTSecretName()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + s, err := k8sutil.GetTokenSecret(ctxChild, secrets, apiObject.Spec.Authentication.GetJWTSecretName()) + cancel() if err != nil { return nil, errors.WithStack(err) } diff --git a/pkg/util/arangod/dbserver.go b/pkg/util/arangod/dbserver.go index 6e422132a..317d88443 100644 --- a/pkg/util/arangod/dbserver.go +++ b/pkg/util/arangod/dbserver.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package arangod @@ -36,16 +37,24 @@ import ( // The functions returns an error when the check could not be completed or the dbserver // is not empty, or nil when the dbserver is found to be empty. func IsDBServerEmpty(ctx context.Context, id string, client driver.Client) error { - c, err := client.Cluster(ctx) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + c, err := client.Cluster(ctxChild) + cancel() if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot obtain Cluster")) } - dbs, err := client.Databases(ctx) + + ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) + dbs, err := client.Databases(ctxChild) + cancel() if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot fetch databases")) } + for _, db := range dbs { - inventory, err := c.DatabaseInventory(ctx, db) + ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) + inventory, err := c.DatabaseInventory(ctxChild, db) + cancel() if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot fetch inventory for %s", db.Name())) } diff --git a/pkg/util/k8sutil/finalizers.go b/pkg/util/k8sutil/finalizers.go index bc88d7c08..4f956b273 100644 --- a/pkg/util/k8sutil/finalizers.go +++ b/pkg/util/k8sutil/finalizers.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -25,11 +26,12 @@ package k8sutil import ( "context" - "github.com/arangodb/kube-arangodb/pkg/util/errors" "github.com/rs/zerolog" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/kubernetes" + + "github.com/arangodb/kube-arangodb/pkg/util/errors" ) const ( @@ -37,10 +39,14 @@ const ( ) // RemovePodFinalizers removes the given finalizers from the given pod. -func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.Pod, finalizers []string, ignoreNotFound bool) error { +func RemovePodFinalizers(ctx context.Context, log zerolog.Logger, kubecli kubernetes.Interface, p *v1.Pod, + finalizers []string, ignoreNotFound bool) error { pods := kubecli.CoreV1().Pods(p.GetNamespace()) getFunc := func() (metav1.Object, error) { - result, err := pods.Get(context.Background(), p.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pods.Get(ctxChild, p.GetName(), metav1.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } @@ -48,7 +54,10 @@ func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1 } updateFunc := func(updated metav1.Object) error { updatedPod := updated.(*v1.Pod) - result, err := pods.Update(context.Background(), updatedPod, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pods.Update(ctxChild, updatedPod, metav1.UpdateOptions{}) if err != nil { return errors.WithStack(err) } @@ -62,10 +71,14 @@ func RemovePodFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1 } // RemovePVCFinalizers removes the given finalizers from the given PVC. -func RemovePVCFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1.PersistentVolumeClaim, finalizers []string, ignoreNotFound bool) error { +func RemovePVCFinalizers(ctx context.Context, log zerolog.Logger, kubecli kubernetes.Interface, + p *v1.PersistentVolumeClaim, finalizers []string, ignoreNotFound bool) error { pvcs := kubecli.CoreV1().PersistentVolumeClaims(p.GetNamespace()) getFunc := func() (metav1.Object, error) { - result, err := pvcs.Get(context.Background(), p.GetName(), metav1.GetOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pvcs.Get(ctxChild, p.GetName(), metav1.GetOptions{}) if err != nil { return nil, errors.WithStack(err) } @@ -73,7 +86,10 @@ func RemovePVCFinalizers(log zerolog.Logger, kubecli kubernetes.Interface, p *v1 } updateFunc := func(updated metav1.Object) error { updatedPVC := updated.(*v1.PersistentVolumeClaim) - result, err := pvcs.Update(context.Background(), updatedPVC, metav1.UpdateOptions{}) + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + result, err := pvcs.Update(ctxChild, updatedPVC, metav1.UpdateOptions{}) if err != nil { return errors.WithStack(err) } diff --git a/pkg/util/k8sutil/inspector/inspector.go b/pkg/util/k8sutil/inspector/inspector.go index a111d78d5..5f11f3fe7 100644 --- a/pkg/util/k8sutil/inspector/inspector.go +++ b/pkg/util/k8sutil/inspector/inspector.go @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Adam Janikowski +// Author Tomasz Mielech // package inspector import ( + "context" + "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/arangomember" "github.com/arangodb/kube-arangodb/pkg/util/k8sutil/inspector/persistentvolumeclaim" @@ -37,7 +40,8 @@ import ( ) type Inspector interface { - Refresh(k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, c versioned.Interface, namespace string) error + Refresh(ctx context.Context, k kubernetes.Interface, m monitoringClient.MonitoringV1Interface, + c versioned.Interface, namespace string) error pod.Inspector secret.Inspector diff --git a/pkg/util/k8sutil/pods.go b/pkg/util/k8sutil/pods.go index 37da41bdb..49a1cd366 100644 --- a/pkg/util/k8sutil/pods.go +++ b/pkg/util/k8sutil/pods.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -406,10 +407,10 @@ func GetPodSpecChecksum(podSpec core.PodSpec) (string, error) { // CreatePod adds an owner to the given pod and calls the k8s api-server to created it. // If the pod already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePod(kubecli kubernetes.Interface, pod *core.Pod, ns string, owner metav1.OwnerReference) (types.UID, error) { +func CreatePod(ctx context.Context, kubecli kubernetes.Interface, pod *core.Pod, ns string, owner metav1.OwnerReference) (types.UID, error) { AddOwnerRefToObject(pod.GetObjectMeta(), &owner) - if pod, err := kubecli.CoreV1().Pods(ns).Create(context.Background(), pod, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { + if pod, err := kubecli.CoreV1().Pods(ns).Create(ctx, pod, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { return "", errors.WithStack(err) } else { return pod.UID, nil diff --git a/pkg/util/k8sutil/pvc.go b/pkg/util/k8sutil/pvc.go index c203ff93d..e8be8949a 100644 --- a/pkg/util/k8sutil/pvc.go +++ b/pkg/util/k8sutil/pvc.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -84,7 +85,7 @@ func ExtractStorageResourceRequirement(resources v1.ResourceRequirements) v1.Res // CreatePersistentVolumeClaim creates a persistent volume claim with given name and configuration. // If the pvc already exists, nil is returned. // If another error occurs, that error is returned. -func CreatePersistentVolumeClaim(pvcs PersistentVolumeClaimInterface, pvcName, deploymentName, ns, storageClassName, role string, enforceAntiAffinity bool, resources v1.ResourceRequirements, vct *v1.PersistentVolumeClaim, finalizers []string, owner metav1.OwnerReference) error { +func CreatePersistentVolumeClaim(ctx context.Context, pvcs PersistentVolumeClaimInterface, pvcName, deploymentName, ns, storageClassName, role string, enforceAntiAffinity bool, resources v1.ResourceRequirements, vct *v1.PersistentVolumeClaim, finalizers []string, owner metav1.OwnerReference) error { labels := LabelsForDeployment(deploymentName, role) volumeMode := v1.PersistentVolumeFilesystem pvc := &v1.PersistentVolumeClaim{ @@ -113,7 +114,7 @@ func CreatePersistentVolumeClaim(pvcs PersistentVolumeClaimInterface, pvcName, d pvc.Spec.StorageClassName = &storageClassName } AddOwnerRefToObject(pvc.GetObjectMeta(), &owner) - if _, err := pvcs.Create(context.Background(), pvc, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { + if _, err := pvcs.Create(ctx, pvc, metav1.CreateOptions{}); err != nil && !IsAlreadyExists(err) { return errors.WithStack(err) } return nil diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 1bb219a70..95386cc2c 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -88,8 +89,8 @@ func CreateEncryptionKeySecret(secrets SecretInterface, secretName string, key [ // ValidateCACertificateSecret checks that a secret with given name in given namespace // exists and it contains a 'ca.crt' data field. -func ValidateCACertificateSecret(secrets SecretInterface, secretName string) error { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func ValidateCACertificateSecret(ctx context.Context, secrets SecretInterface, secretName string) error { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -106,8 +107,11 @@ func ValidateCACertificateSecret(secrets SecretInterface, secretName string) err // If the secret does not exists the field is missing, // an error is returned. // Returns: certificate, error -func GetCACertficateSecret(secrets SecretInterface, secretName string) (string, error) { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func GetCACertficateSecret(ctx context.Context, secrets SecretInterface, secretName string) (string, error) { + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + + s, err := secrets.Get(ctxChild, secretName, meta.GetOptions{}) if err != nil { return "", errors.WithStack(err) } @@ -124,8 +128,9 @@ func GetCACertficateSecret(secrets SecretInterface, secretName string) (string, // If the secret does not exists or one of the fields is missing, // an error is returned. // Returns: certificate, private-key, isOwnedByDeployment, error -func GetCASecret(secrets SecretInterface, secretName string, ownerRef *meta.OwnerReference) (string, string, bool, error) { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func GetCASecret(ctx context.Context, secrets SecretInterface, secretName string, + ownerRef *meta.OwnerReference) (string, string, bool, error) { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return "", "", false, errors.WithStack(err) } @@ -155,7 +160,8 @@ func GetCAFromSecret(s *core.Secret, ownerRef *meta.OwnerReference) (string, str } // CreateCASecret creates a secret used to store a PEM encoded CA certificate & private key. -func CreateCASecret(secrets SecretInterface, secretName string, certificate, key string, ownerRef *meta.OwnerReference) error { +func CreateCASecret(ctx context.Context, secrets SecretInterface, secretName string, certificate, key string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -168,7 +174,7 @@ func CreateCASecret(secrets SecretInterface, secretName string, certificate, key } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -197,7 +203,8 @@ func GetTLSKeyfileFromSecret(s *core.Secret) (string, error) { // CreateTLSKeyfileSecret creates a secret used to store a PEM encoded keyfile // in the format ArangoDB accepts it for its `--ssl.keyfile` option. -func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile string, ownerRef *meta.OwnerReference) error { +func CreateTLSKeyfileSecret(ctx context.Context, secrets SecretInterface, secretName string, keyfile string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -209,7 +216,7 @@ func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -218,8 +225,8 @@ func CreateTLSKeyfileSecret(secrets SecretInterface, secretName string, keyfile // ValidateTokenSecret checks that a secret with given name in given namespace // exists and it contains a 'token' data field. -func ValidateTokenSecret(secrets SecretInterface, secretName string) error { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func ValidateTokenSecret(ctx context.Context, secrets SecretInterface, secretName string) error { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return errors.WithStack(err) } @@ -236,8 +243,8 @@ func ValidateTokenFromSecret(s *core.Secret) error { } // GetTokenSecret loads the token secret from a Secret with given name. -func GetTokenSecret(secrets SecretInterface, secretName string) (string, error) { - s, err := secrets.Get(context.Background(), secretName, meta.GetOptions{}) +func GetTokenSecret(ctx context.Context, secrets SecretInterface, secretName string) (string, error) { + s, err := secrets.Get(ctx, secretName, meta.GetOptions{}) if err != nil { return "", errors.WithStack(err) } @@ -256,7 +263,8 @@ func GetTokenFromSecret(s *core.Secret) (string, error) { // CreateTokenSecret creates a secret with given name in given namespace // with a given token as value. -func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerRef *meta.OwnerReference) error { +func CreateTokenSecret(ctx context.Context, secrets SecretInterface, secretName, token string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -268,7 +276,7 @@ func CreateTokenSecret(secrets SecretInterface, secretName, token string, ownerR } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + if _, err := secrets.Create(ctx, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } @@ -292,8 +300,10 @@ func CreateJWTTokenFromSecret(secret string, claims map[string]interface{}) (str // CreateJWTFromSecret creates a JWT using the secret stored in secretSecretName and stores the // result in a new secret called tokenSecretName -func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error { - secret, err := GetTokenSecret(secrets, secretSecretName) +func CreateJWTFromSecret(ctx context.Context, secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error { + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + secret, err := GetTokenSecret(ctxChild, secrets, secretSecretName) + cancel() if err != nil { return errors.WithStack(err) } @@ -307,12 +317,15 @@ func CreateJWTFromSecret(secrets SecretInterface, tokenSecretName, secretSecretN return errors.WithStack(err) } - return CreateTokenSecret(secrets, tokenSecretName, signedToken, ownerRef) + ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + return CreateTokenSecret(ctxChild, secrets, tokenSecretName, signedToken, ownerRef) } // CreateBasicAuthSecret creates a secret with given name in given namespace // with a given username and password as value. -func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, password string, ownerRef *meta.OwnerReference) error { +func CreateBasicAuthSecret(ctx context.Context, secrets SecretInterface, secretName, username, password string, + ownerRef *meta.OwnerReference) error { // Create secret secret := &core.Secret{ ObjectMeta: meta.ObjectMeta{ @@ -325,7 +338,9 @@ func CreateBasicAuthSecret(secrets SecretInterface, secretName, username, passwo } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - if _, err := secrets.Create(context.Background(), secret, meta.CreateOptions{}); err != nil { + ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() + if _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}); err != nil { // Failed to create secret return errors.WithStack(err) } diff --git a/pkg/util/k8sutil/services.go b/pkg/util/k8sutil/services.go index ddd1f1ab2..ee8e379ae 100644 --- a/pkg/util/k8sutil/services.go +++ b/pkg/util/k8sutil/services.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil @@ -76,7 +77,8 @@ func CreateExporterClientServiceName(deploymentName string) string { } // CreateExporterService -func CreateExporterService(cachedStatus service.Inspector, svcs ServiceInterface, deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { +func CreateExporterService(ctx context.Context, cachedStatus service.Inspector, svcs ServiceInterface, + deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() svcName := CreateExporterClientServiceName(deploymentName) @@ -104,7 +106,7 @@ func CreateExporterService(cachedStatus service.Inspector, svcs ServiceInterface }, } AddOwnerRefToObject(svc.GetObjectMeta(), &owner) - if _, err := svcs.Create(context.Background(), svc, metav1.CreateOptions{}); IsAlreadyExists(err) { + if _, err := svcs.Create(ctx, svc, metav1.CreateOptions{}); IsAlreadyExists(err) { return svcName, false, nil } else if err != nil { return svcName, false, errors.WithStack(err) @@ -117,7 +119,8 @@ func CreateExporterService(cachedStatus service.Inspector, svcs ServiceInterface // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateHeadlessService(svcs ServiceInterface, deployment metav1.Object, owner metav1.OwnerReference) (string, bool, error) { +func CreateHeadlessService(ctx context.Context, svcs ServiceInterface, deployment metav1.Object, + owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() svcName := CreateHeadlessServiceName(deploymentName) ports := []core.ServicePort{ @@ -129,7 +132,7 @@ func CreateHeadlessService(svcs ServiceInterface, deployment metav1.Object, owne } publishNotReadyAddresses := true serviceType := core.ServiceTypeClusterIP - newlyCreated, err := createService(svcs, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", serviceType, ports, "", nil, publishNotReadyAddresses, owner) + newlyCreated, err := createService(ctx, svcs, svcName, deploymentName, deployment.GetNamespace(), ClusterIPNone, "", serviceType, ports, "", nil, publishNotReadyAddresses, owner) if err != nil { return "", false, errors.WithStack(err) } @@ -140,7 +143,8 @@ func CreateHeadlessService(svcs ServiceInterface, deployment metav1.Object, owne // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateDatabaseClientService(svcs ServiceInterface, deployment metav1.Object, single bool, owner metav1.OwnerReference) (string, bool, error) { +func CreateDatabaseClientService(ctx context.Context, svcs ServiceInterface, deployment metav1.Object, single bool, + owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() svcName := CreateDatabaseClientServiceName(deploymentName) ports := []core.ServicePort{ @@ -158,7 +162,7 @@ func CreateDatabaseClientService(svcs ServiceInterface, deployment metav1.Object } serviceType := core.ServiceTypeClusterIP publishNotReadyAddresses := false - newlyCreated, err := createService(svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, "", nil, publishNotReadyAddresses, owner) + newlyCreated, err := createService(ctx, svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, "", nil, publishNotReadyAddresses, owner) if err != nil { return "", false, errors.WithStack(err) } @@ -169,7 +173,9 @@ func CreateDatabaseClientService(svcs ServiceInterface, deployment metav1.Object // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func CreateExternalAccessService(svcs ServiceInterface, svcName, role string, deployment metav1.Object, serviceType core.ServiceType, port, nodePort int, loadBalancerIP string, loadBalancerSourceRanges []string, owner metav1.OwnerReference) (string, bool, error) { +func CreateExternalAccessService(ctx context.Context, svcs ServiceInterface, svcName, role string, + deployment metav1.Object, serviceType core.ServiceType, port, nodePort int, loadBalancerIP string, + loadBalancerSourceRanges []string, owner metav1.OwnerReference) (string, bool, error) { deploymentName := deployment.GetName() ports := []core.ServicePort{ core.ServicePort{ @@ -180,7 +186,7 @@ func CreateExternalAccessService(svcs ServiceInterface, svcName, role string, de }, } publishNotReadyAddresses := false - newlyCreated, err := createService(svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, loadBalancerIP, loadBalancerSourceRanges, publishNotReadyAddresses, owner) + newlyCreated, err := createService(ctx, svcs, svcName, deploymentName, deployment.GetNamespace(), "", role, serviceType, ports, loadBalancerIP, loadBalancerSourceRanges, publishNotReadyAddresses, owner) if err != nil { return "", false, errors.WithStack(err) } @@ -191,8 +197,9 @@ func CreateExternalAccessService(svcs ServiceInterface, svcName, role string, de // If the service already exists, nil is returned. // If another error occurs, that error is returned. // The returned bool is true if the service is created, or false when the service already existed. -func createService(svcs ServiceInterface, svcName, deploymentName, ns, clusterIP, role string, serviceType core.ServiceType, - ports []core.ServicePort, loadBalancerIP string, loadBalancerSourceRanges []string, publishNotReadyAddresses bool, owner metav1.OwnerReference) (bool, error) { +func createService(ctx context.Context, svcs ServiceInterface, svcName, deploymentName, ns, clusterIP, role string, + serviceType core.ServiceType, ports []core.ServicePort, loadBalancerIP string, loadBalancerSourceRanges []string, + publishNotReadyAddresses bool, owner metav1.OwnerReference) (bool, error) { labels := LabelsForDeployment(deploymentName, role) svc := &core.Service{ ObjectMeta: metav1.ObjectMeta{ @@ -211,7 +218,7 @@ func createService(svcs ServiceInterface, svcName, deploymentName, ns, clusterIP }, } AddOwnerRefToObject(svc.GetObjectMeta(), &owner) - if _, err := svcs.Create(context.Background(), svc, metav1.CreateOptions{}); IsAlreadyExists(err) { + if _, err := svcs.Create(ctx, svc, metav1.CreateOptions{}); IsAlreadyExists(err) { return false, nil } else if err != nil { return false, errors.WithStack(err) diff --git a/pkg/util/k8sutil/util.go b/pkg/util/k8sutil/util.go index f9891154a..7df03b01e 100644 --- a/pkg/util/k8sutil/util.go +++ b/pkg/util/k8sutil/util.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,11 +18,14 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package k8sutil import ( + "time" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) @@ -43,8 +46,25 @@ const ( // AppName is the fixed value for the "app" label AppName = "arangodb" + + // minDefaultRequestTimeout is minimum default request timeout to k8s. + minDefaultRequestTimeout = time.Second * 3 ) +var requestTimeout = minDefaultRequestTimeout + +// GetRequestTimeout gets request timeout for one call to kubernetes. +func GetRequestTimeout() time.Duration { + return requestTimeout +} + +// SetRequestTimeout sets request timeout for one call to kubernetes. +func SetRequestTimeout(timeout time.Duration) { + if timeout > minDefaultRequestTimeout { + requestTimeout = timeout + } +} + // AddOwnerRefToObject adds given owner reference to given object func AddOwnerRefToObject(obj metav1.Object, ownerRef *metav1.OwnerReference) { if ownerRef != nil { diff --git a/tests/auth_test.go b/tests/auth_test.go index 5aa6c9fc7..b6390bbbb 100644 --- a/tests/auth_test.go +++ b/tests/auth_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package tests @@ -102,7 +103,7 @@ func TestAuthenticationSingleCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateTokenSecret(secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(context.Background(), secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } defer removeSecret(kubecli, depl.Spec.Authentication.GetJWTSecretName(), ns) @@ -152,7 +153,7 @@ func TestAuthenticationNoneSingle(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create deployment - apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl,metav1.CreateOptions{}) + apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl, metav1.CreateOptions{}) if err != nil { t.Fatalf("Create deployment failed: %v", err) } @@ -240,12 +241,12 @@ func TestAuthenticationClusterCustomSecret(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create secret - if err := k8sutil.CreateTokenSecret(secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { + if err := k8sutil.CreateTokenSecret(context.Background(), secrets, depl.Spec.Authentication.GetJWTSecretName(), "foo", nil); err != nil { t.Fatalf("Create JWT secret failed: %v", err) } // Create deployment - apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl,metav1.CreateOptions{}) + apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl, metav1.CreateOptions{}) if err != nil { t.Fatalf("Create deployment failed: %v", err) } @@ -292,7 +293,7 @@ func TestAuthenticationNoneCluster(t *testing.T) { depl.Spec.SetDefaults(depl.GetName()) // Create deployment - apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl,metav1.CreateOptions{}) + apiObject, err := c.DatabaseV1().ArangoDeployments(ns).Create(context.Background(), depl, metav1.CreateOptions{}) if err != nil { t.Fatalf("Create deployment failed: %v", err) } diff --git a/tests/sync/main.go b/tests/sync/main.go index ceb6c47ae..8aa5df8fb 100644 --- a/tests/sync/main.go +++ b/tests/sync/main.go @@ -143,7 +143,7 @@ func newArangoSyncTestJob(ns, name string) *batchv1.Job { func waitForSyncDeploymentReady(ctx context.Context, ns, name string, kubecli kubernetes.Interface, c versioned.Interface) error { return retry.Retry(func() error { - deployment, err := c.DatabaseV1().ArangoDeployments(ns).Get(name, metav1.GetOptions{}) + deployment, err := c.DatabaseV1().ArangoDeployments(ns).Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } @@ -420,7 +420,7 @@ func mustNewArangoDBSyncClient(ctx context.Context, kubecli kubernetes.Interface ns := deployment.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := deployment.Spec.Sync.Authentication.GetJWTSecretName() - jwtSecret, err := k8sutil.GetTokenSecret(secrets, secretName) + jwtSecret, err := k8sutil.GetTokenSecret(ctx, secrets, secretName) if err != nil { return nil, err } diff --git a/tests/test_util.go b/tests/test_util.go index 964c65d94..6ce3a6167 100644 --- a/tests/test_util.go +++ b/tests/test_util.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2020 ArangoDB GmbH, Cologne, Germany +// Copyright 2020-2021 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -18,6 +18,7 @@ // Copyright holder is ArangoDB GmbH, Cologne, Germany // // Author Ewout Prangsma +// Author Tomasz Mielech // package tests @@ -217,7 +218,7 @@ func mustNewArangoSyncClient(ctx context.Context, kubecli kubernetes.Interface, ns := apiObject.GetNamespace() secrets := kubecli.CoreV1().Secrets(ns) secretName := apiObject.Spec.Sync.Authentication.GetJWTSecretName() - jwtToken, err := k8sutil.GetTokenSecret(secrets, secretName) + jwtToken, err := k8sutil.GetTokenSecret(ctx, secrets, secretName) if err != nil { t.Fatalf("Failed to get sync jwt secret '%s': %s", secretName, err) } From cab2a2eedb78f6ade59a916e6f45fd1a73d26940 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Tue, 20 Apr 2021 11:08:44 +0200 Subject: [PATCH 2/6] increase timeout for the restore backup action --- pkg/deployment/reconcile/action_backup_restore.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/deployment/reconcile/action_backup_restore.go b/pkg/deployment/reconcile/action_backup_restore.go index aff976de4..b8d80f857 100644 --- a/pkg/deployment/reconcile/action_backup_restore.go +++ b/pkg/deployment/reconcile/action_backup_restore.go @@ -98,9 +98,8 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { return false, err } - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - restoreError := dbc.Backup().Restore(ctxChild, driver.BackupID(backupResource.Status.Backup.ID), nil) - cancel() + // The below action can take a while so the full parent timeout context is used. + restoreError := dbc.Backup().Restore(ctx, driver.BackupID(backupResource.Status.Backup.ID), nil) if restoreError != nil { a.log.Error().Err(restoreError).Msg("Restore failed") } From 7e7fff349e5892aa0c0578a40e415b854c046cd4 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Wed, 21 Apr 2021 14:31:34 +0200 Subject: [PATCH 3/6] clean go.sum --- go.sum | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/go.sum b/go.sum index 6c0a38a3f..62d0f273e 100644 --- a/go.sum +++ b/go.sum @@ -93,6 +93,7 @@ github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfc github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -195,6 +196,7 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= @@ -208,6 +210,7 @@ github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4er github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= @@ -234,6 +237,7 @@ github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v0.0.0-20161122191042-44d81051d367/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= @@ -249,6 +253,7 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -280,6 +285,7 @@ github.com/jessevdk/go-assets-builder v0.0.0-20130903091706-b8483521738f/go.mod github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v0.0.0-20180612202835-f2b4162afba3/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= @@ -334,6 +340,7 @@ github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/f github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180320133207-05fbef0ca5da/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -363,6 +370,7 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= @@ -397,6 +405,7 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ= github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= @@ -480,7 +489,9 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190125153040-c74c464bbbf2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190312203227-4b39c73a6495/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= @@ -501,6 +512,7 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -522,6 +534,7 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= @@ -541,6 +554,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -571,6 +585,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd h1:5CtCZbICpIOFdgO940moixOPjc0178IU44m4EjOO5IY= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.0.0-20160726164857-2910a502d2bf/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -590,6 +605,7 @@ golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -607,6 +623,7 @@ golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -618,6 +635,9 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.0.0-20190331200053-3d26580ed485/go.mod h1:2ltnJ7xHfj0zHS40VVPYEAAMTa3ZGguvHGBSJeRWqE0= +gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= +gonum.org/v1/netlib v0.0.0-20190331212654-76723241ea4e/go.mod h1:kS+toOQn6AQKjmKJ7gzohV1XkqsFehRA2FbsbkopSuQ= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -708,19 +728,29 @@ k8s.io/apiserver v0.19.8/go.mod h1:+OJE9rJCT99Qr9DYITQDCKDLxFLVi5zA8nI9KqjGshk= k8s.io/client-go v0.19.8 h1:rcb2BrXb1HUBiBCoP3m/9Q2VZIMWhZUAmH49EmAyRUA= k8s.io/client-go v0.19.8/go.mod h1:5Op2bSbK+COBz8mwH62rrRgqhA9wOcORkWZ03+GL0Ow= k8s.io/component-base v0.19.8/go.mod h1:cvHAT4oGxKsfcnnm0hMp3JkEMxAt5s6le943V796FXM= +k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20190822140433-26a664648505/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A= k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200410163147-594e756bea31/go.mod h1:1TqjTSzOxsLGIKfj0lK8EeCP7K1iUG65v09OM0/WG5E= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6 h1:+WnxoVtG8TMiudHBSEtrVL1egv36TkkJm+bA8AxicmQ= k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= k8s.io/utils v0.0.0-20200729134348-d5654de09c73 h1:uJmqzgNWG7XyClnU/mLPBWwfKKF1K8Hf8whTseBgJcg= k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= +modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= +modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= +modernc.org/strutil v1.0.0/go.mod h1:lstksw84oURvj9y3tn8lGvRxyRC1S2+g5uuIzNfIOBs= +modernc.org/xc v1.0.0/go.mod h1:mRNCo0bvLjGhHO9WsyuKVU4q0ceiDDDoEeWDJHrNx8I= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQbTRyDlZPJX2SUPEqvnB+j7AJjtlox7PEwigU0= +sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= sigs.k8s.io/structured-merge-diff/v4 v4.0.1 h1:YXTMot5Qz/X1iBRJhAt+vI+HVttY0WkSqqhKxQ0xVbA= sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0 h1:4A07+ZFc2wgJwo8YNlQpr1rVlgUDlxXHhPJciaPY5gs= From 00cdddd277a8cb2fa698506c6e38856264858c46 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Thu, 22 Apr 2021 07:09:59 +0200 Subject: [PATCH 4/6] small improvements after the code review --- pkg/backup/handlers/arango/backup/arango_client_impl.go | 2 +- pkg/deployment/deployment_inspector.go | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/backup/handlers/arango/backup/arango_client_impl.go b/pkg/backup/handlers/arango/backup/arango_client_impl.go index 8055c2397..7a47f74a0 100644 --- a/pkg/backup/handlers/arango/backup/arango_client_impl.go +++ b/pkg/backup/handlers/arango/backup/arango_client_impl.go @@ -131,8 +131,8 @@ func (ac *arangoClientBackupImpl) Get(backupID driver.BackupID) (driver.BackupMe func (ac *arangoClientBackupImpl) getCredentialsFromSecret(ctx context.Context, secretName string) (interface{}, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() token, err := k8sutil.GetTokenSecret(ctxChild, ac.kubecli.CoreV1().Secrets(ac.backup.Namespace), secretName) - cancel() if err != nil { return nil, err } diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index e8092a90e..1c3000f11 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -47,7 +47,7 @@ import ( ) const ( - twentySeconds = time.Second * 20 + timeoutReconciliationPerNode = time.Second * 20 ) var ( @@ -65,7 +65,7 @@ func (d *Deployment) getReconciliationTimeout() (time.Duration, error) { return 0, errors.Wrapf(err, "Unable to get nodes") } - if timeout := twentySeconds * time.Duration(len(nodes.Items)); timeout > time.Minute { + if timeout := timeoutReconciliationPerNode * time.Duration(len(nodes.Items)); timeout > time.Minute { return timeout, nil } @@ -91,8 +91,8 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval } ctxReconciliation, cancelReconciliation := context.WithTimeout(context.Background(), timeout) + defer cancelReconciliation() defer func() { - cancelReconciliation() d.deps.Log.Info().Msgf("Inspect loop took %s", time.Since(start)) }() From dda444dcbb1f5baea6198d8140fbdc5ae842a3f9 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Thu, 22 Apr 2021 14:41:55 +0200 Subject: [PATCH 5/6] use defer cancel() in most cases --- pkg/deployment/access_package.go | 29 +++---- pkg/deployment/cluster_scaling_integration.go | 6 +- pkg/deployment/context_impl.go | 41 +++++----- pkg/deployment/deployment.go | 13 ++-- pkg/deployment/deployment_finalizers.go | 2 +- pkg/deployment/images.go | 25 +++--- .../reconcile/action_backup_restore.go | 2 +- .../action_bootstrap_set_password.go | 30 +++++--- .../reconcile/action_cleanout_member.go | 16 ++-- .../action_cluster_member_cleanup.go | 17 ++-- .../reconcile/action_encryption_add.go | 8 +- .../reconcile/action_encryption_refresh.go | 6 +- .../reconcile/action_encryption_remove.go | 8 +- pkg/deployment/reconcile/action_jwt_add.go | 8 +- pkg/deployment/reconcile/action_jwt_clean.go | 8 +- .../reconcile/action_jwt_refresh.go | 2 +- .../reconcile/action_jwt_set_active.go | 8 +- .../reconcile/action_maintenance_disable.go | 9 ++- .../reconcile/action_maintenance_enable.go | 9 ++- .../reconcile/action_remove_member.go | 2 +- .../reconcile/action_resign_leadership.go | 10 +-- .../reconcile/action_tls_ca_append.go | 8 +- .../reconcile/action_tls_ca_clean.go | 8 +- .../reconcile/action_tls_ca_renew.go | 10 +-- .../reconcile/action_tls_keyfile_refresh.go | 4 +- .../reconcile/action_tls_sni_update.go | 2 +- .../reconcile/action_tls_status_update.go | 2 +- pkg/deployment/reconcile/helper_shutdown.go | 2 +- .../reconcile/plan_builder_cluster.go | 6 +- .../reconcile/plan_builder_common.go | 4 +- .../reconcile/plan_builder_encryption.go | 4 +- pkg/deployment/resilience/member_failure.go | 4 +- pkg/deployment/resources/annotations.go | 77 +++++++++---------- pkg/deployment/resources/certificates_tls.go | 9 ++- pkg/deployment/resources/inspector/members.go | 2 +- pkg/deployment/resources/inspector/pdbs.go | 2 +- pkg/deployment/resources/inspector/pods.go | 2 +- pkg/deployment/resources/inspector/pvcs.go | 2 +- pkg/deployment/resources/inspector/sa.go | 2 +- pkg/deployment/resources/inspector/secrets.go | 2 +- .../resources/inspector/services.go | 2 +- pkg/deployment/resources/inspector/sms.go | 2 +- pkg/deployment/resources/labels.go | 77 +++++++++---------- pkg/deployment/resources/member_cleanup.go | 13 ++-- pkg/deployment/resources/pdbs.go | 13 ++-- pkg/deployment/resources/pod_creator.go | 29 ++++--- pkg/deployment/resources/pod_finalizers.go | 12 +-- pkg/deployment/resources/pod_termination.go | 22 +++--- pkg/deployment/resources/pvc_finalizers.go | 6 +- pkg/deployment/resources/pvcs.go | 6 +- pkg/deployment/resources/secrets.go | 53 +++++++------ pkg/deployment/resources/servicemonitor.go | 22 +++--- pkg/deployment/resources/services.go | 35 +++++---- pkg/util/arangod/client.go | 14 +++- pkg/util/arangod/dbserver.go | 4 +- pkg/util/k8sutil/secrets.go | 16 ++-- pkg/util/k8sutil/util.go | 16 ++++ 57 files changed, 393 insertions(+), 360 deletions(-) diff --git a/pkg/deployment/access_package.go b/pkg/deployment/access_package.go index 3b7e1d8c4..f67987f26 100644 --- a/pkg/deployment/access_package.go +++ b/pkg/deployment/access_package.go @@ -68,8 +68,8 @@ func (d *Deployment) createAccessPackages(ctx context.Context) error { // Remove all access packages that we did build, but are no longer needed ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() secretList, err := secrets.List(ctxChild, metav1.ListOptions{}) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to list secrets") return errors.WithStack(err) @@ -80,11 +80,11 @@ func (d *Deployment) createAccessPackages(ctx context.Context) error { // Secret is an access package if _, wanted := apNameMap[secret.GetName()]; !wanted { // We found an obsolete access package secret. Remove it. - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := secrets.Delete(ctxChild, secret.GetName(), metav1.DeleteOptions{ - Preconditions: &metav1.Preconditions{UID: &secret.UID}, + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return secrets.Delete(ctxChild, secret.GetName(), metav1.DeleteOptions{ + Preconditions: &metav1.Preconditions{UID: &secret.UID}, + }) }) - cancel() if err != nil && !k8sutil.IsNotFound(err) { // Not serious enough to stop everything now, just log and create an event log.Warn().Err(err).Msg("Failed to remove obsolete access package secret") @@ -110,9 +110,10 @@ func (d *Deployment) ensureAccessPackage(ctx context.Context, apSecretName strin secrets := d.deps.KubeCli.CoreV1().Secrets(ns) spec := d.apiObject.Spec - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := secrets.Get(ctxChild, apSecretName, metav1.GetOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Get(ctxChild, apSecretName, metav1.GetOptions{}) + return err + }) if err == nil { // Secret already exists return nil @@ -123,9 +124,9 @@ func (d *Deployment) ensureAccessPackage(ctx context.Context, apSecretName strin // Fetch client authentication CA clientAuthSecretName := spec.Sync.Authentication.GetClientCASecretName() - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() clientAuthCert, clientAuthKey, _, err := k8sutil.GetCASecret(ctxChild, secrets, clientAuthSecretName, nil) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to get client-auth CA secret") return errors.WithStack(err) @@ -219,9 +220,11 @@ func (d *Deployment) ensureAccessPackage(ctx context.Context, apSecretName strin } // Attach secret to owner secret.SetOwnerReferences(append(secret.GetOwnerReferences(), d.apiObject.AsOwner())) - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - if _, err := secrets.Create(ctxChild, secret, metav1.CreateOptions{}); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Create(ctxChild, secret, metav1.CreateOptions{}) + return err + }) + if err != nil { // Failed to create secret log.Debug().Err(err).Str("secret-name", apSecretName).Msg("Failed to create access package Secret") return errors.WithStack(err) diff --git a/pkg/deployment/cluster_scaling_integration.go b/pkg/deployment/cluster_scaling_integration.go index 4ba09f1a6..3cb9ffe9a 100644 --- a/pkg/deployment/cluster_scaling_integration.go +++ b/pkg/deployment/cluster_scaling_integration.go @@ -148,15 +148,15 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectS log := ci.log ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := ci.depl.clientCache.GetDatabase(ctxChild) - cancel() if err != nil { return errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() req, err := arangod.GetNumberOfServers(ctxChild, c.Connection()) - cancel() if err != nil { if expectSuccess { log.Debug().Err(err).Msg("Failed to get number of servers") @@ -200,8 +200,8 @@ func (ci *clusterScalingIntegration) inspectCluster(ctx context.Context, expectS // Let's update the spec apiObject := ci.depl.apiObject ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() current, err := ci.depl.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(apiObject.Namespace).Get(ctxChild, apiObject.Name, metav1.GetOptions{}) - cancel() if err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/context_impl.go b/pkg/deployment/context_impl.go index c158555a0..867a68d83 100644 --- a/pkg/deployment/context_impl.go +++ b/pkg/deployment/context_impl.go @@ -393,10 +393,10 @@ func (d *Deployment) GetPod(ctx context.Context, podName string) (*v1.Pod, error func (d *Deployment) DeletePod(ctx context.Context, podName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - if err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, meta.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to remove pod") return errors.WithStack(err) } @@ -411,10 +411,9 @@ func (d *Deployment) CleanupPod(ctx context.Context, p *v1.Pod) error { ns := p.GetNamespace() options := meta.NewDeleteOptions(0) options.Preconditions = meta.NewUIDPreconditions(string(p.GetUID())) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - err := d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, *options) + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, *options) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pod", podName).Msg("Failed to cleanup pod") return errors.WithStack(err) @@ -430,8 +429,8 @@ func (d *Deployment) RemovePodFinalizers(ctx context.Context, podName string) er kubecli := d.deps.KubeCli ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() p, err := kubecli.CoreV1().Pods(ns).Get(ctxChild, podName, meta.GetOptions{}) - cancel() if err != nil { if k8sutil.IsNotFound(err) { return nil @@ -451,10 +450,10 @@ func (d *Deployment) RemovePodFinalizers(ctx context.Context, podName string) er func (d *Deployment) DeletePvc(ctx context.Context, pvcName string) error { log := d.deps.Log ns := d.apiObject.GetNamespace() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - if err := d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(ctxChild, pvcName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().PersistentVolumeClaims(ns).Delete(ctxChild, pvcName, meta.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Str("pvc", pvcName).Msg("Failed to remove pvc") return errors.WithStack(err) } @@ -464,10 +463,10 @@ func (d *Deployment) DeletePvc(ctx context.Context, pvcName string) error { // UpdatePvc updated a persistent volume claim in the namespace // of the deployment. If the pvc does not exist, the error is ignored. func (d *Deployment) UpdatePvc(ctx context.Context, pvc *v1.PersistentVolumeClaim) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(ctxChild, pvc, meta.UpdateOptions{}) + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := d.GetKubeCli().CoreV1().PersistentVolumeClaims(d.GetNamespace()).Update(ctxChild, pvc, meta.UpdateOptions{}) + return err + }) if err == nil { return nil } @@ -528,10 +527,10 @@ func (d *Deployment) GetTLSKeyfile(group api.ServerGroup, member api.MemberStatu func (d *Deployment) DeleteTLSKeyfile(ctx context.Context, group api.ServerGroup, member api.MemberStatus) error { secretName := k8sutil.CreateTLSKeyfileSecretName(d.apiObject.GetName(), group.AsRole(), member.ID) ns := d.apiObject.GetNamespace() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - if err := d.deps.KubeCli.CoreV1().Secrets(ns).Delete(ctxChild, secretName, meta.DeleteOptions{}); err != nil && !k8sutil.IsNotFound(err) { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return d.deps.KubeCli.CoreV1().Secrets(ns).Delete(ctxChild, secretName, meta.DeleteOptions{}) + }) + if err != nil && !k8sutil.IsNotFound(err) { return errors.WithStack(err) } return nil diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index dadcd4235..ce9d5dfe4 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -304,8 +304,8 @@ func (d *Deployment) handleArangoDeploymentUpdatedEvent(ctx context.Context) err // Get the most recent version of the deployment from the API server ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() current, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctxChild, d.apiObject.GetName(), metav1.GetOptions{}) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to get current version of deployment from API server") if k8sutil.IsNotFound(err) { @@ -518,15 +518,16 @@ func (d *Deployment) lookForServiceMonitorCRD() { // SetNumberOfServers adjust number of DBservers and coordinators in arangod func (d *Deployment) SetNumberOfServers(ctx context.Context, noCoordinators, noDBServers *int) error { ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := d.clientCache.GetDatabase(ctxChild) - cancel() if err != nil { return errors.WithStack(err) } - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - err = arangod.SetNumberOfServers(ctxChild, c.Connection(), noCoordinators, noDBServers) - cancel() + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return arangod.SetNumberOfServers(ctxChild, c.Connection(), noCoordinators, noDBServers) + }) + if err != nil { return errors.WithStack(err) } @@ -548,8 +549,8 @@ func (d *Deployment) ApplyPatch(ctx context.Context, p ...patch.Item) error { c := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()) ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() depl, err := c.Patch(ctxChild, d.apiObject.GetName(), types.JSONPatchType, data, metav1.PatchOptions{}) - cancel() if err != nil { return err } diff --git a/pkg/deployment/deployment_finalizers.go b/pkg/deployment/deployment_finalizers.go index 8e2375e1a..8229e0beb 100644 --- a/pkg/deployment/deployment_finalizers.go +++ b/pkg/deployment/deployment_finalizers.go @@ -58,8 +58,8 @@ func (d *Deployment) runDeploymentFinalizers(ctx context.Context, cachedStatus i depls := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.GetNamespace()) ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() updated, err := depls.Get(ctxChild, d.apiObject.GetName(), metav1.GetOptions{}) - cancel() if err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/images.go b/pkg/deployment/images.go index a29533467..4763eec31 100644 --- a/pkg/deployment/images.go +++ b/pkg/deployment/images.go @@ -136,16 +136,16 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima // Check if pod exists ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() pod, err := ib.KubeCli.CoreV1().Pods(ns).Get(ctxChild, podName, metav1.GetOptions{}) - cancel() if err == nil { // Pod found if k8sutil.IsPodFailed(pod) { // Wait some time before deleting the pod if time.Now().After(pod.GetCreationTimestamp().Add(30 * time.Second)) { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete Image ID Pod") return false, nil @@ -175,8 +175,8 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima return true, nil } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() v, err := client.Version(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch version from Image ID Pod") return true, nil @@ -185,9 +185,9 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima enterprise := strings.ToLower(v.License) == "enterprise" // We have all the info we need now, kill the pod and store the image info. - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err = ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return ib.KubeCli.CoreV1().Pods(ns).Delete(ctxChild, podName, metav1.DeleteOptions{}) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete Image ID Pod") return true, nil @@ -231,10 +231,11 @@ func (ib *imagesBuilder) fetchArangoDBImageIDAndVersion(ctx context.Context, ima return true, errors.WithStack(err) } - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - if _, err := resources.CreateArangoPod(ctxChild, ib.KubeCli, ib.APIObject, ib.Spec, api.ServerGroupImageDiscovery, pod); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := resources.CreateArangoPod(ctxChild, ib.KubeCli, ib.APIObject, ib.Spec, api.ServerGroupImageDiscovery, pod) + return err + }) + if err != nil { log.Debug().Err(err).Msg("Failed to create image ID pod") return true, errors.WithStack(err) } diff --git a/pkg/deployment/reconcile/action_backup_restore.go b/pkg/deployment/reconcile/action_backup_restore.go index b8d80f857..3fddd3f0e 100644 --- a/pkg/deployment/reconcile/action_backup_restore.go +++ b/pkg/deployment/reconcile/action_backup_restore.go @@ -67,8 +67,8 @@ func (a actionBackupRestore) Start(ctx context.Context) (bool, error) { } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() dbc, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_bootstrap_set_password.go b/pkg/deployment/reconcile/action_bootstrap_set_password.go index e50ea7d4b..720aab826 100644 --- a/pkg/deployment/reconcile/action_bootstrap_set_password.go +++ b/pkg/deployment/reconcile/action_bootstrap_set_password.go @@ -104,8 +104,8 @@ func (a actionBootstrapSetPassword) setUserPassword(ctx context.Context, user, s a.log.Debug().Msgf("Bootstrapping user %s, secret %s", user, secret) ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { return "", errors.WithStack(err) } @@ -116,21 +116,27 @@ func (a actionBootstrapSetPassword) setUserPassword(ctx context.Context, user, s } // Obtain the user - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - u, err := client.User(ctxChild, user) - cancel() - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) defer cancel() - if driver.IsNotFound(err) { - _, err := client.CreateUser(ctxChild, user, &driver.UserOptions{Password: password}) + if u, err := client.User(ctxChild, user); err != nil { + if !driver.IsNotFound(err) { + return "", err + } + + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := client.CreateUser(ctxChild, user, &driver.UserOptions{Password: password}) + return err + }) + return password, errors.WithStack(err) - } else if err == nil { - return password, errors.WithStack(u.Update(ctxChild, driver.UserOptions{ - Password: password, - })) } else { - return "", err + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return u.Update(ctxChild, driver.UserOptions{ + Password: password, + }) + }) + + return password, errors.WithStack(err) } } diff --git a/pkg/deployment/reconcile/action_cleanout_member.go b/pkg/deployment/reconcile/action_cleanout_member.go index 2f4417263..e2847b5ab 100644 --- a/pkg/deployment/reconcile/action_cleanout_member.go +++ b/pkg/deployment/reconcile/action_cleanout_member.go @@ -67,16 +67,16 @@ func (a *actionCleanoutMember) Start(ctx context.Context) (bool, error) { log := a.log ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cluster, err := c.Cluster(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") return false, errors.WithStack(err) @@ -115,24 +115,24 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create database client") return false, false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cluster, err := c.Cluster(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") return false, false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cleanedOut, err := cluster.IsCleanedOut(ctxChild, a.action.MemberID) - cancel() if err != nil { log.Debug().Err(err).Msg("IsCleanedOut failed") return false, false, errors.WithStack(err) @@ -142,24 +142,24 @@ func (a *actionCleanoutMember) CheckProgress(ctx context.Context) (bool, bool, e log.Debug().Msg("IsCleanedOut returned false") ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create database client") return false, false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() agency, err := a.actionCtx.GetAgency(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return false, false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch cleanout job status") return false, false, errors.WithStack(err) diff --git a/pkg/deployment/reconcile/action_cluster_member_cleanup.go b/pkg/deployment/reconcile/action_cluster_member_cleanup.go index feb08d3e4..eb7447aa7 100644 --- a/pkg/deployment/reconcile/action_cluster_member_cleanup.go +++ b/pkg/deployment/reconcile/action_cluster_member_cleanup.go @@ -72,22 +72,22 @@ func (a *actionClusterMemberCleanup) start(ctx context.Context) error { id := driver.ServerID(a.MemberID()) ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { return err } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cluster, err := c.Cluster(ctxChild) - cancel() if err != nil { return err } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() health, err := cluster.Health(ctxChild) - cancel() if err != nil { return err } @@ -96,12 +96,7 @@ func (a *actionClusterMemberCleanup) start(ctx context.Context) error { return nil } - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - defer cancel() - - if err := cluster.RemoveServer(ctxChild, id); err != nil { - return err - } - - return nil + return arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return cluster.RemoveServer(ctxChild, id) + }) } diff --git a/pkg/deployment/reconcile/action_encryption_add.go b/pkg/deployment/reconcile/action_encryption_add.go index 62a83f479..db051f952 100644 --- a/pkg/deployment/reconcile/action_encryption_add.go +++ b/pkg/deployment/reconcile/action_encryption_add.go @@ -107,10 +107,10 @@ func (a *encryptionKeyAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { return false, err } diff --git a/pkg/deployment/reconcile/action_encryption_refresh.go b/pkg/deployment/reconcile/action_encryption_refresh.go index acb44df4a..ae2fde2a6 100644 --- a/pkg/deployment/reconcile/action_encryption_refresh.go +++ b/pkg/deployment/reconcile/action_encryption_refresh.go @@ -59,16 +59,16 @@ func (a *encryptionKeyRefreshAction) Start(ctx context.Context) (bool, error) { func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() keyfolder, err := a.actionCtx.SecretsInterface().Get(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetName()), meta.GetOptions{}) - cancel() if err != nil { a.log.Err(err).Msgf("Unable to fetch encryption folder") return true, false, nil } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) - cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil @@ -76,8 +76,8 @@ func (a *encryptionKeyRefreshAction) CheckProgress(ctx context.Context) (bool, b client := client.NewClient(c.Connection()) ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() e, err := client.RefreshEncryption(ctxChild) - cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to refresh encryption") return true, false, nil diff --git a/pkg/deployment/reconcile/action_encryption_remove.go b/pkg/deployment/reconcile/action_encryption_remove.go index 9b7d44af5..7e80ae06c 100644 --- a/pkg/deployment/reconcile/action_encryption_remove.go +++ b/pkg/deployment/reconcile/action_encryption_remove.go @@ -82,10 +82,10 @@ func (a *encryptionKeyRemoveAction) Start(ctx context.Context) (bool, error) { return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.GetEncryptionFolderSecretName(a.actionCtx.GetAPIObject().GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_jwt_add.go b/pkg/deployment/reconcile/action_jwt_add.go index dc2fd210c..03859700b 100644 --- a/pkg/deployment/reconcile/action_jwt_add.go +++ b/pkg/deployment/reconcile/action_jwt_add.go @@ -117,10 +117,10 @@ func (a *jwtAddAction) Start(ctx context.Context) (bool, error) { return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_clean.go b/pkg/deployment/reconcile/action_jwt_clean.go index b27b143f6..cdc675316 100644 --- a/pkg/deployment/reconcile/action_jwt_clean.go +++ b/pkg/deployment/reconcile/action_jwt_clean.go @@ -108,10 +108,10 @@ func (a *jwtCleanAction) Start(ctx context.Context) (bool, error) { return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_jwt_refresh.go b/pkg/deployment/reconcile/action_jwt_refresh.go index 71b2dcdca..4a9792dae 100644 --- a/pkg/deployment/reconcile/action_jwt_refresh.go +++ b/pkg/deployment/reconcile/action_jwt_refresh.go @@ -61,8 +61,8 @@ func (a *jwtRefreshAction) CheckProgress(ctx context.Context) (bool, bool, error } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) - cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil diff --git a/pkg/deployment/reconcile/action_jwt_set_active.go b/pkg/deployment/reconcile/action_jwt_set_active.go index ae7a88fd4..823ffdcc9 100644 --- a/pkg/deployment/reconcile/action_jwt_set_active.go +++ b/pkg/deployment/reconcile/action_jwt_set_active.go @@ -119,10 +119,10 @@ func (a *jwtSetActiveAction) Start(ctx context.Context) (bool, error) { return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, pod.JWTSecretFolder(a.actionCtx.GetName()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", pod.JWTSecretFolder(a.actionCtx.GetName())) diff --git a/pkg/deployment/reconcile/action_maintenance_disable.go b/pkg/deployment/reconcile/action_maintenance_disable.go index 2927a9de0..988fdde0d 100644 --- a/pkg/deployment/reconcile/action_maintenance_disable.go +++ b/pkg/deployment/reconcile/action_maintenance_disable.go @@ -60,16 +60,17 @@ func (a *actionDisableMaintenance) Start(ctx context.Context) (bool, error) { } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { a.log.Error().Err(err).Msgf("Unable to get agency client") return true, nil } - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - defer cancel() - if err := agency.SetMaintenanceMode(ctxChild, client, false); err != nil { + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return agency.SetMaintenanceMode(ctxChild, client, false) + }) + if err != nil { a.log.Error().Err(err).Msgf("Unable to disable maintenance") return true, nil } diff --git a/pkg/deployment/reconcile/action_maintenance_enable.go b/pkg/deployment/reconcile/action_maintenance_enable.go index d032f63e0..1110b074a 100644 --- a/pkg/deployment/reconcile/action_maintenance_enable.go +++ b/pkg/deployment/reconcile/action_maintenance_enable.go @@ -61,16 +61,17 @@ func (a *actionEnableMaintenance) Start(ctx context.Context) (bool, error) { } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { a.log.Error().Err(err).Msgf("Unable to get agency client") return true, nil } - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - defer cancel() - if err := agency.SetMaintenanceMode(ctxChild, client, true); err != nil { + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return agency.SetMaintenanceMode(ctxChild, client, true) + }) + if err != nil { a.log.Error().Err(err).Msgf("Unable to set maintenance") return true, nil } diff --git a/pkg/deployment/reconcile/action_remove_member.go b/pkg/deployment/reconcile/action_remove_member.go index 28e2958a6..5959458f3 100644 --- a/pkg/deployment/reconcile/action_remove_member.go +++ b/pkg/deployment/reconcile/action_remove_member.go @@ -70,8 +70,8 @@ func (a *actionRemoveMember) Start(ctx context.Context) (bool, error) { // For safety, remove from cluster if a.action.Group == api.ServerGroupCoordinators || a.action.Group == api.ServerGroupDBServers { ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { return false, errors.WithStack(err) } diff --git a/pkg/deployment/reconcile/action_resign_leadership.go b/pkg/deployment/reconcile/action_resign_leadership.go index bef0fd301..0f372136d 100644 --- a/pkg/deployment/reconcile/action_resign_leadership.go +++ b/pkg/deployment/reconcile/action_resign_leadership.go @@ -70,8 +70,8 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get client") return true, errors.WithStack(err) @@ -80,8 +80,8 @@ func (a *actionResignLeadership) Start(ctx context.Context) (bool, error) { switch group { case api.ServerGroupDBServers: ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cluster, err := client.Cluster(ctxChild) - cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get cluster client") return true, errors.WithStack(err) @@ -120,24 +120,24 @@ func (a *actionResignLeadership) CheckProgress(ctx context.Context) (bool, bool, } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() agency, err := a.actionCtx.GetAgency(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return false, false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, false, errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, m.CleanoutJobID, c, agency) - cancel() if err != nil { if driver.IsNotFound(err) { log.Debug().Err(err).Msg("Job not found, but proceeding") diff --git a/pkg/deployment/reconcile/action_tls_ca_append.go b/pkg/deployment/reconcile/action_tls_ca_append.go index 2724a212d..7e60e830f 100644 --- a/pkg/deployment/reconcile/action_tls_ca_append.go +++ b/pkg/deployment/reconcile/action_tls_ca_append.go @@ -116,10 +116,10 @@ func (a *appendTLSCACertificateAction) Start(ctx context.Context) (bool, error) return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_tls_ca_clean.go b/pkg/deployment/reconcile/action_tls_ca_clean.go index a587420c9..c5651c7c1 100644 --- a/pkg/deployment/reconcile/action_tls_ca_clean.go +++ b/pkg/deployment/reconcile/action_tls_ca_clean.go @@ -119,10 +119,10 @@ func (a *cleanTLSCACertificateAction) Start(ctx context.Context) (bool, error) { a.log.Info().Msgf("Removing key %s from truststore", certChecksum) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err = a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := a.actionCtx.SecretsInterface().Patch(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), types.JSONPatchType, patch, meta.PatchOptions{}) + return err + }) if err != nil { if !k8sutil.IsInvalid(err) { return false, errors.Wrapf(err, "Unable to update secret: %s", string(patch)) diff --git a/pkg/deployment/reconcile/action_tls_ca_renew.go b/pkg/deployment/reconcile/action_tls_ca_renew.go index 8cb646ddf..b7ebb2514 100644 --- a/pkg/deployment/reconcile/action_tls_ca_renew.go +++ b/pkg/deployment/reconcile/action_tls_ca_renew.go @@ -55,11 +55,11 @@ func (a *renewTLSCACertificateAction) Start(ctx context.Context) (bool, error) { return true, nil } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - s := a.actionCtx.SecretsInterface() - if err := s.Delete(ctxChild, a.actionCtx.GetSpec().TLS.GetCASecretName(), meta.DeleteOptions{}); err != nil { + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + s := a.actionCtx.SecretsInterface() + return s.Delete(ctxChild, a.actionCtx.GetSpec().TLS.GetCASecretName(), meta.DeleteOptions{}) + }) + if err != nil { if !k8sutil.IsNotFound(err) { a.log.Warn().Err(err).Msgf("Unable to clean cert %s", a.actionCtx.GetSpec().TLS.GetCASecretName()) return true, nil diff --git a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go index 96cea6f3d..d15eec61c 100644 --- a/pkg/deployment/reconcile/action_tls_keyfile_refresh.go +++ b/pkg/deployment/reconcile/action_tls_keyfile_refresh.go @@ -55,8 +55,8 @@ type refreshTLSKeyfileCertificateAction struct { func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) (bool, bool, error) { ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := a.actionCtx.GetServerClient(ctxChild, a.action.Group, a.action.MemberID) - cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil @@ -79,8 +79,8 @@ func (a *refreshTLSKeyfileCertificateAction) CheckProgress(ctx context.Context) client := client.NewClient(c.Connection()) ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() e, err := client.RefreshTLS(ctxChild) - cancel() if err != nil { a.log.Warn().Err(err).Msg("Unable to refresh TLS") return true, false, nil diff --git a/pkg/deployment/reconcile/action_tls_sni_update.go b/pkg/deployment/reconcile/action_tls_sni_update.go index d930cbaf4..f5ff7dc71 100644 --- a/pkg/deployment/reconcile/action_tls_sni_update.go +++ b/pkg/deployment/reconcile/action_tls_sni_update.go @@ -71,8 +71,8 @@ func (t *tlsSNIUpdate) CheckProgress(ctx context.Context) (bool, bool, error) { } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := t.actionCtx.GetServerClient(ctxChild, t.action.Group, t.action.MemberID) - cancel() if err != nil { t.log.Warn().Err(err).Msg("Unable to get client") return true, false, nil diff --git a/pkg/deployment/reconcile/action_tls_status_update.go b/pkg/deployment/reconcile/action_tls_status_update.go index 4d70135e2..22bae8d17 100644 --- a/pkg/deployment/reconcile/action_tls_status_update.go +++ b/pkg/deployment/reconcile/action_tls_status_update.go @@ -59,8 +59,8 @@ func (a *tlsKeyStatusUpdateAction) Start(ctx context.Context) (bool, error) { } ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() f, err := a.actionCtx.SecretsInterface().Get(ctxChild, resources.GetCASecretName(a.actionCtx.GetAPIObject()), meta.GetOptions{}) - cancel() if err != nil { a.log.Error().Err(err).Msgf("Unable to get folder info") return true, nil diff --git a/pkg/deployment/reconcile/helper_shutdown.go b/pkg/deployment/reconcile/helper_shutdown.go index 445764ab0..5557efebd 100644 --- a/pkg/deployment/reconcile/helper_shutdown.go +++ b/pkg/deployment/reconcile/helper_shutdown.go @@ -68,8 +68,8 @@ func (s shutdownHelperAPI) Start(ctx context.Context) (bool, error) { if group.IsArangod() { // Invoke shutdown endpoint ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := s.actionCtx.GetServerClient(ctxChild, group, s.action.MemberID) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return false, errors.WithStack(err) diff --git a/pkg/deployment/reconcile/plan_builder_cluster.go b/pkg/deployment/reconcile/plan_builder_cluster.go index eef43e29c..5acb825c0 100644 --- a/pkg/deployment/reconcile/plan_builder_cluster.go +++ b/pkg/deployment/reconcile/plan_builder_cluster.go @@ -48,23 +48,23 @@ func createClusterOperationPlan(ctx context.Context, } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := planCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { return nil } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cluster, err := c.Cluster(ctxChild) - cancel() if err != nil { log.Warn().Err(err).Msgf("Unable to get Cluster client") return nil } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() health, err := cluster.Health(ctxChild) - cancel() if err != nil { log.Warn().Err(err).Msgf("Unable to get Cluster health") return nil diff --git a/pkg/deployment/reconcile/plan_builder_common.go b/pkg/deployment/reconcile/plan_builder_common.go index fcdbf9373..0aede0445 100644 --- a/pkg/deployment/reconcile/plan_builder_common.go +++ b/pkg/deployment/reconcile/plan_builder_common.go @@ -50,16 +50,16 @@ func createMaintenanceManagementPlan(ctx context.Context, } ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := planCtx.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get agency client") return nil } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() m, err := agency.GetMaintenanceMode(ctxChild, client) - cancel() if err != nil { log.Error().Err(err).Msgf("Unable to get agency maintenance mode") return nil diff --git a/pkg/deployment/reconcile/plan_builder_encryption.go b/pkg/deployment/reconcile/plan_builder_encryption.go index 1dcc68d1d..37229190c 100644 --- a/pkg/deployment/reconcile/plan_builder_encryption.go +++ b/pkg/deployment/reconcile/plan_builder_encryption.go @@ -285,8 +285,8 @@ func isEncryptionKeyUpToDate(ctx context.Context, mlog := log.With().Str("group", group.AsRole()).Str("member", m.ID).Logger() ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := planCtx.GetServerClient(ctxChild, group, m.ID) - cancel() if err != nil { mlog.Warn().Err(err).Msg("Unable to get client") return false, true @@ -295,8 +295,8 @@ func isEncryptionKeyUpToDate(ctx context.Context, client := client.NewClient(c.Connection()) ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() e, err := client.GetEncryption(ctxChild) - cancel() if err != nil { mlog.Error().Err(err).Msgf("Unable to fetch encryption keys") return false, true diff --git a/pkg/deployment/resilience/member_failure.go b/pkg/deployment/resilience/member_failure.go index 05c8e39d7..9eb415131 100644 --- a/pkg/deployment/resilience/member_failure.go +++ b/pkg/deployment/resilience/member_failure.go @@ -133,8 +133,8 @@ func (r *Resilience) isMemberFailureAcceptable(ctx context.Context, group api.Se case api.ServerGroupAgents: // All good when remaining agents are health ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() clients, err := r.context.GetAgencyClients(ctxChild, func(id string) bool { return id != m.ID }) - cancel() if err != nil { return false, "", errors.WithStack(err) } @@ -144,8 +144,8 @@ func (r *Resilience) isMemberFailureAcceptable(ctx context.Context, group api.Se return true, "", nil case api.ServerGroupDBServers: ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() client, err := r.context.GetDatabaseClient(ctxChild) - cancel() if err != nil { return false, "", errors.WithStack(err) } diff --git a/pkg/deployment/resources/annotations.go b/pkg/deployment/resources/annotations.go index e87701d44..ba0994a33 100644 --- a/pkg/deployment/resources/annotations.go +++ b/pkg/deployment/resources/annotations.go @@ -50,12 +50,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto log.Info().Msgf("Ensuring annotations") patchSecret := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := kubecli.CoreV1().Secrets(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, - meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().Secrets(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) } if err := ensureSecretsAnnotations(patchSecret, @@ -68,12 +67,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto } patchServiceAccount := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, - meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().ServiceAccounts(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) } if err := ensureServiceAccountsAnnotations(patchServiceAccount, @@ -86,12 +84,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto } patchService := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := kubecli.CoreV1().Services(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, - meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().Services(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) } if err := ensureServicesAnnotations(patchService, @@ -104,12 +101,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto } patchPDB := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()).Patch(ctxChild, name, - types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.PolicyV1beta1().PodDisruptionBudgets(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) } if err := ensurePdbsAnnotations(patchPDB, @@ -122,12 +118,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto } patchPVC := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Patch(ctxChild, name, - types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().PersistentVolumeClaims(r.context.GetNamespace()).Patch(ctxChild, name, + types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) } if err := ensurePvcsAnnotations(patchPVC, @@ -140,12 +135,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto } patchPod := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := kubecli.CoreV1().Pods(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, - meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := kubecli.CoreV1().Pods(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) } if err := ensurePodsAnnotations(patchPod, @@ -159,12 +153,11 @@ func (r *Resources) EnsureAnnotations(ctx context.Context, cachedStatus inspecto } patchServiceMonitor := func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := monitoringcli.ServiceMonitors(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, - meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := monitoringcli.ServiceMonitors(r.context.GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, + meta.PatchOptions{}) + return err + }) } if err := ensureServiceMonitorsAnnotations(patchServiceMonitor, diff --git a/pkg/deployment/resources/certificates_tls.go b/pkg/deployment/resources/certificates_tls.go index be0ff2b0a..b4c54ebc1 100644 --- a/pkg/deployment/resources/certificates_tls.go +++ b/pkg/deployment/resources/certificates_tls.go @@ -90,8 +90,8 @@ func createTLSServerCertificate(ctx context.Context, log zerolog.Logger, secrets // Load CA certificate ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() caCert, caKey, _, err := k8sutil.GetCASecret(ctxChild, secrets, spec.GetCASecretName(), nil) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to load CA certificate") return errors.WithStack(err) @@ -118,10 +118,11 @@ func createTLSServerCertificate(ctx context.Context, log zerolog.Logger, secrets } keyfile := strings.TrimSpace(cert) + "\n" + strings.TrimSpace(priv) - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - if err := k8sutil.CreateTLSKeyfileSecret(ctxChild, secrets, secretName, keyfile, ownerRef); err != nil { + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreateTLSKeyfileSecret(ctxChild, secrets, secretName, keyfile, ownerRef) + }) + if err != nil { if k8sutil.IsAlreadyExists(err) { log.Debug().Msg("Server Secret already exists") } else { diff --git a/pkg/deployment/resources/inspector/members.go b/pkg/deployment/resources/inspector/members.go index 7ca95dcca..8d5ef7b9f 100644 --- a/pkg/deployment/resources/inspector/members.go +++ b/pkg/deployment/resources/inspector/members.go @@ -104,11 +104,11 @@ func arangoMemberPointer(pod api.ArangoMember) *api.ArangoMember { func getArangoMembers(ctx context.Context, k versioned.Interface, namespace, cont string) ([]api.ArangoMember, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() arangoMembers, err := k.DatabaseV1().ArangoMembers(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/pdbs.go b/pkg/deployment/resources/inspector/pdbs.go index 6a1a7727e..a7f6c0263 100644 --- a/pkg/deployment/resources/inspector/pdbs.go +++ b/pkg/deployment/resources/inspector/pdbs.go @@ -104,11 +104,11 @@ func podDisruptionBudgetPointer(podDisruptionBudget policy.PodDisruptionBudget) func getPodDisruptionBudgets(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]policy.PodDisruptionBudget, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() podDisruptionBudgets, err := k.PolicyV1beta1().PodDisruptionBudgets(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/pods.go b/pkg/deployment/resources/inspector/pods.go index ba2c511d7..1f2eb514b 100644 --- a/pkg/deployment/resources/inspector/pods.go +++ b/pkg/deployment/resources/inspector/pods.go @@ -103,11 +103,11 @@ func podPointer(pod core.Pod) *core.Pod { func getPods(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Pod, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() pods, err := k.CoreV1().Pods(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/pvcs.go b/pkg/deployment/resources/inspector/pvcs.go index dae5b2000..ea83cf8c6 100644 --- a/pkg/deployment/resources/inspector/pvcs.go +++ b/pkg/deployment/resources/inspector/pvcs.go @@ -104,11 +104,11 @@ func pvcPointer(pvc core.PersistentVolumeClaim) *core.PersistentVolumeClaim { func getPersistentVolumeClaims(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.PersistentVolumeClaim, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() pvcs, err := k.CoreV1().PersistentVolumeClaims(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/sa.go b/pkg/deployment/resources/inspector/sa.go index aac7e4a11..8ae14e07d 100644 --- a/pkg/deployment/resources/inspector/sa.go +++ b/pkg/deployment/resources/inspector/sa.go @@ -104,11 +104,11 @@ func serviceAccountPointer(serviceAccount core.ServiceAccount) *core.ServiceAcco func getServiceAccounts(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.ServiceAccount, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() serviceAccounts, err := k.CoreV1().ServiceAccounts(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/secrets.go b/pkg/deployment/resources/inspector/secrets.go index 4450d7125..7c3ca8cfa 100644 --- a/pkg/deployment/resources/inspector/secrets.go +++ b/pkg/deployment/resources/inspector/secrets.go @@ -125,11 +125,11 @@ func secretPointer(pod core.Secret) *core.Secret { func getSecrets(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Secret, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() secrets, err := k.CoreV1().Secrets(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/services.go b/pkg/deployment/resources/inspector/services.go index 5ccfc85b5..f60977b84 100644 --- a/pkg/deployment/resources/inspector/services.go +++ b/pkg/deployment/resources/inspector/services.go @@ -104,11 +104,11 @@ func servicePointer(pod core.Service) *core.Service { func getServices(ctx context.Context, k kubernetes.Interface, namespace, cont string) ([]core.Service, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() services, err := k.CoreV1().Services(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return nil, err diff --git a/pkg/deployment/resources/inspector/sms.go b/pkg/deployment/resources/inspector/sms.go index 381d27f15..3a6df419e 100644 --- a/pkg/deployment/resources/inspector/sms.go +++ b/pkg/deployment/resources/inspector/sms.go @@ -100,11 +100,11 @@ func serviceMonitorsToMap(ctx context.Context, m monitoringClient.MonitoringV1In func getServiceMonitors(ctx context.Context, m monitoringClient.MonitoringV1Interface, namespace, cont string) ([]*monitoring.ServiceMonitor, error) { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() serviceMonitors, err := m.ServiceMonitors(namespace).List(ctxChild, meta.ListOptions{ Limit: 128, Continue: cont, }) - cancel() if err != nil { return []*monitoring.ServiceMonitor{}, nil diff --git a/pkg/deployment/resources/labels.go b/pkg/deployment/resources/labels.go index 8d5b45b6b..4fffcdfa7 100644 --- a/pkg/deployment/resources/labels.go +++ b/pkg/deployment/resources/labels.go @@ -74,12 +74,11 @@ func (r *Resources) EnsureSecretLabels(ctx context.Context, cachedStatus inspect changed := false if err := cachedStatus.IterateSecrets(func(secret *core.Secret) error { if ensureLabelsMap(secret.Kind, secret, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, - name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().Secrets(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -102,12 +101,11 @@ func (r *Resources) EnsureServiceAccountsLabels(ctx context.Context, cachedStatu changed := false if err := cachedStatus.IterateServiceAccounts(func(serviceAccount *core.ServiceAccount) error { if ensureLabelsMap(serviceAccount.Kind, serviceAccount, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetKubeCli().CoreV1().ServiceAccounts(r.context.GetAPIObject().GetNamespace()). - Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().ServiceAccounts(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -130,12 +128,11 @@ func (r *Resources) EnsureServicesLabels(ctx context.Context, cachedStatus inspe changed := false if err := cachedStatus.IterateServices(func(service *core.Service) error { if ensureLabelsMap(service.Kind, service, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetKubeCli().CoreV1().Services(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, - name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().Services(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -158,12 +155,11 @@ func (r *Resources) EnsureServiceMonitorsLabels(ctx context.Context, cachedStatu changed := false if err := cachedStatus.IterateServiceMonitors(func(serviceMonitor *monitoring.ServiceMonitor) error { if ensureLabelsMap(serviceMonitor.Kind, serviceMonitor, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()). - Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetMonitoringV1Cli().ServiceMonitors(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -186,12 +182,11 @@ func (r *Resources) EnsurePodsLabels(ctx context.Context, cachedStatus inspector changed := false if err := cachedStatus.IteratePods(func(pod *core.Pod) error { if ensureGroupLabelsMap(pod.Kind, pod, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetKubeCli().CoreV1().Pods(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, - name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().Pods(r.context.GetAPIObject().GetNamespace()).Patch(ctxChild, + name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -214,12 +209,11 @@ func (r *Resources) EnsurePersistentVolumeClaimsLabels(ctx context.Context, cach changed := false if err := cachedStatus.IteratePersistentVolumeClaims(func(persistentVolumeClaim *core.PersistentVolumeClaim) error { if ensureGroupLabelsMap(persistentVolumeClaim.Kind, persistentVolumeClaim, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetAPIObject().GetNamespace()). - Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetAPIObject().GetNamespace()). + Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } @@ -242,12 +236,11 @@ func (r *Resources) EnsurePodDisruptionBudgetsLabels(ctx context.Context, cached changed := false if err := cachedStatus.IteratePodDisruptionBudgets(func(budget *policy.PodDisruptionBudget) error { if ensureLabelsMap(budget.Kind, budget, r.context.GetSpec(), func(name string, d []byte) error { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - defer cancel() - - _, err := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetAPIObject(). - GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) - return err + return k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetKubeCli().PolicyV1beta1().PodDisruptionBudgets(r.context.GetAPIObject(). + GetNamespace()).Patch(ctxChild, name, types.JSONPatchType, d, meta.PatchOptions{}) + return err + }) }) { changed = true } diff --git a/pkg/deployment/resources/member_cleanup.go b/pkg/deployment/resources/member_cleanup.go index 6e531b9ae..15cc09fb6 100644 --- a/pkg/deployment/resources/member_cleanup.go +++ b/pkg/deployment/resources/member_cleanup.go @@ -188,9 +188,10 @@ func (r *Resources) EnsureArangoMembers(ctx context.Context, cachedStatus inspec }, } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Create(ctxChild, &a, metav1.CreateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Create(ctxChild, &a, metav1.CreateOptions{}) + return err + }) if err != nil { return err } @@ -210,9 +211,9 @@ func (r *Resources) EnsureArangoMembers(ctx context.Context, cachedStatus inspec if !ok || g != member.Spec.Group { // Remove member - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Delete(ctxChild, member.GetName(), metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return r.context.GetArangoCli().DatabaseV1().ArangoMembers(obj.GetNamespace()).Delete(ctxChild, member.GetName(), metav1.DeleteOptions{}) + }) if err != nil { if !k8sutil.IsNotFound(err) { return err diff --git a/pkg/deployment/resources/pdbs.go b/pkg/deployment/resources/pdbs.go index 0c07a1fda..dc81dacec 100644 --- a/pkg/deployment/resources/pdbs.go +++ b/pkg/deployment/resources/pdbs.go @@ -121,9 +121,10 @@ func (r *Resources) ensurePDBForGroup(ctx context.Context, group api.ServerGroup // No PDB found - create new pdb := newPDB(wantedMinAvail, deplname, group, r.context.GetAPIObject().AsOwner()) log.Debug().Msg("Creating new PDB") - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := pdbcli.Create(ctxChild, pdb, metav1.CreateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := pdbcli.Create(ctxChild, pdb, metav1.CreateOptions{}) + return err + }) if err != nil { log.Error().Err(err).Msg("failed to create PDB") return errors.WithStack(err) @@ -145,9 +146,9 @@ func (r *Resources) ensurePDBForGroup(ctx context.Context, group api.ServerGroup // Trigger deletion only if not already deleted if pdb.GetDeletionTimestamp() == nil { // Update the PDB - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := pdbcli.Delete(ctxChild, pdbname, metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pdbcli.Delete(ctxChild, pdbname, metav1.DeleteOptions{}) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Error().Err(err).Msg("PDB deletion failed") return errors.WithStack(err) diff --git a/pkg/deployment/resources/pod_creator.go b/pkg/deployment/resources/pod_creator.go index c73332bc5..be974f0e2 100644 --- a/pkg/deployment/resources/pod_creator.go +++ b/pkg/deployment/resources/pod_creator.go @@ -369,17 +369,17 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect // Check master JWT secret masterJWTSecretName = spec.Sync.Authentication.GetJWTSecretName() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := k8sutil.ValidateTokenSecret(ctxChild, secrets, masterJWTSecretName) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateTokenSecret(ctxChild, secrets, masterJWTSecretName) + }) if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Master JWT secret validation failed")) } monitoringTokenSecretName := spec.Sync.Monitoring.GetTokenSecretName() - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err = k8sutil.ValidateTokenSecret(ctxChild, secrets, monitoringTokenSecretName) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateTokenSecret(ctxChild, secrets, monitoringTokenSecretName) + }) if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Monitoring token secret validation failed")) } @@ -390,19 +390,18 @@ func (r *Resources) RenderPodForMember(ctx context.Context, cachedStatus inspect // Check cluster JWT secret if spec.IsAuthenticated() { clusterJWTSecretName = spec.Authentication.GetJWTSecretName() - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err = k8sutil.ValidateTokenSecret(ctxChild, secrets, clusterJWTSecretName) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateTokenSecret(ctxChild, secrets, clusterJWTSecretName) + }) if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Cluster JWT secret validation failed")) } } // Check client-auth CA certificate secret clientAuthCASecretName = spec.Sync.Authentication.GetClientCASecretName() - - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err = k8sutil.ValidateCACertificateSecret(ctxChild, secrets, clientAuthCASecretName) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.ValidateCACertificateSecret(ctxChild, secrets, clientAuthCASecretName) + }) if err != nil { return nil, errors.WithStack(errors.Wrapf(err, "Client authentication CA certificate secret validation failed")) } @@ -511,8 +510,8 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS } ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) - cancel() if err != nil { return errors.WithStack(err) } @@ -564,8 +563,8 @@ func (r *Resources) createPodForMember(ctx context.Context, spec api.DeploymentS } ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() uid, err := CreateArangoPod(ctxChild, kubecli, apiObject, spec, group, pod) - cancel() if err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/resources/pod_finalizers.go b/pkg/deployment/resources/pod_finalizers.go index 0ab655e43..2a6ccdef6 100644 --- a/pkg/deployment/resources/pod_finalizers.go +++ b/pkg/deployment/resources/pod_finalizers.go @@ -119,9 +119,9 @@ func (r *Resources) inspectFinalizerPodAgencyServing(ctx context.Context, log ze // of the agent, also remove the PVC if memberStatus.Conditions.IsTrue(api.ConditionTypeAgentRecoveryNeeded) { pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace()) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete PVC for member") return errors.WithStack(err) @@ -149,9 +149,9 @@ func (r *Resources) inspectFinalizerPodDrainDBServer(ctx context.Context, log ze // If this DBServer is cleaned out, we need to remove the PVC. if memberStatus.Conditions.IsTrue(api.ConditionTypeCleanedOut) || memberStatus.Phase == api.MemberPhaseDrain { pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(r.context.GetNamespace()) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pvcs.Delete(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.DeleteOptions{}) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Warn().Err(err).Msg("Failed to delete PVC for member") return errors.WithStack(err) diff --git a/pkg/deployment/resources/pod_termination.go b/pkg/deployment/resources/pod_termination.go index 081cc5f1e..346d13ca6 100644 --- a/pkg/deployment/resources/pod_termination.go +++ b/pkg/deployment/resources/pod_termination.go @@ -60,8 +60,8 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog agentDataWillBeGone := false if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(ctxChild, p.Spec.NodeName, metav1.GetOptions{}) - cancel() if k8sutil.IsNotFound(err) { log.Warn().Msg("Node not found") } else if err != nil { @@ -75,8 +75,8 @@ func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog // Check PVC pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() pvc, err := pvcs.Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) - cancel() if err != nil { log.Warn().Err(err).Msg("Failed to get PVC for member") return errors.WithStack(err) @@ -159,8 +159,8 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol dbserverDataWillBeGone := false if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() node, err := r.context.GetKubeCli().CoreV1().Nodes().Get(ctxChild, p.Spec.NodeName, metav1.GetOptions{}) - cancel() if k8sutil.IsNotFound(err) { log.Warn().Msg("Node not found") } else if err != nil { @@ -176,8 +176,8 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol // Check PVC pvcs := r.context.GetKubeCli().CoreV1().PersistentVolumeClaims(apiObject.GetNamespace()) ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() pvc, err := pvcs.Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{}) - cancel() if err != nil { log.Warn().Err(err).Msg("Failed to get PVC for member") return errors.WithStack(err) @@ -199,15 +199,15 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol // Inspect cleaned out state ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() c, err := r.context.GetDatabaseClient(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create member client") return errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cluster, err := c.Cluster(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to access cluster") @@ -221,8 +221,8 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol return errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() cleanedOut, err := cluster.IsCleanedOut(ctxChild, memberStatus.ID) - cancel() if err != nil { return errors.WithStack(err) } @@ -289,15 +289,15 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } else if memberStatus.Phase == api.MemberPhaseDrain { // Check the job progress ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() agency, err := r.context.GetAgency(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch job status") return errors.WithStack(err) @@ -320,16 +320,16 @@ func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerol } else if memberStatus.Phase == api.MemberPhaseResign { // Check the job progress ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() agency, err := r.context.GetAgency(ctxChild) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create agency client") return errors.WithStack(err) } ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) + defer cancel() jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to fetch job status") return errors.WithStack(err) diff --git a/pkg/deployment/resources/pvc_finalizers.go b/pkg/deployment/resources/pvc_finalizers.go index 011cf47c2..a230aadad 100644 --- a/pkg/deployment/resources/pvc_finalizers.go +++ b/pkg/deployment/resources/pvc_finalizers.go @@ -108,9 +108,9 @@ func (r *Resources) inspectFinalizerPVCMemberExists(ctx context.Context, log zer if memberStatus.PodName != "" { log.Info().Msg("Removing Pod of member, because PVC is being removed") pods := r.context.GetKubeCli().CoreV1().Pods(apiObject.GetNamespace()) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := pods.Delete(ctxChild, memberStatus.PodName, metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return pods.Delete(ctxChild, memberStatus.PodName, metav1.DeleteOptions{}) + }) if err != nil && !k8sutil.IsNotFound(err) { log.Debug().Err(err).Msg("Failed to delete pod") return errors.WithStack(err) diff --git a/pkg/deployment/resources/pvcs.go b/pkg/deployment/resources/pvcs.go index 4876d9d44..2ef141630 100644 --- a/pkg/deployment/resources/pvcs.go +++ b/pkg/deployment/resources/pvcs.go @@ -65,9 +65,9 @@ func (r *Resources) EnsurePVCs(ctx context.Context, cachedStatus inspectorInterf resources := spec.Resources vct := spec.VolumeClaimTemplate finalizers := r.createPVCFinalizers(group) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := k8sutil.CreatePersistentVolumeClaim(ctxChild, pvcs, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, vct, finalizers, owner) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreatePersistentVolumeClaim(ctxChild, pvcs, m.PersistentVolumeClaimName, deploymentName, ns, storageClassName, role, enforceAntiAffinity, resources, vct, finalizers, owner) + }) if err != nil { return errors.WithStack(err) } diff --git a/pkg/deployment/resources/secrets.go b/pkg/deployment/resources/secrets.go index 88b2877e0..93d56cc11 100644 --- a/pkg/deployment/resources/secrets.go +++ b/pkg/deployment/resources/secrets.go @@ -240,9 +240,10 @@ func (r *Resources) ensureTokenSecretFolder(ctx context.Context, cachedStatus in f.Data[pod.ActiveJWTKey] = token f.Data[constants.SecretKeyToken] = token - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := secrets.Update(ctxChild, f, meta.UpdateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Update(ctxChild, f, meta.UpdateOptions{}) + return err + }) if err != nil { return err } @@ -264,10 +265,10 @@ func (r *Resources) ensureTokenSecretFolder(ctx context.Context, cachedStatus in return err } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err = secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) - cancel() - + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) + return err + }) if err != nil { return err } @@ -287,9 +288,10 @@ func (r *Resources) ensureTokenSecretFolder(ctx context.Context, cachedStatus in return err } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err = secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Patch(ctxChild, folderSecretName, types.JSONPatchType, pdata, meta.PatchOptions{}) + return err + }) if err != nil { return err } @@ -383,9 +385,10 @@ func (r *Resources) createSecretWithMod(ctx context.Context, secrets k8sutil.Sec f(secret) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}) + return err + }) if err != nil { // Failed to create secret return errors.WithStack(err) @@ -407,9 +410,9 @@ func (r *Resources) createTokenSecret(ctx context.Context, secrets k8sutil.Secre // Create secret owner := r.context.GetAPIObject().AsOwner() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := k8sutil.CreateTokenSecret(ctxChild, secrets, secretName, token, &owner) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return k8sutil.CreateTokenSecret(ctxChild, secrets, secretName, token, &owner) + }) if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil @@ -448,9 +451,9 @@ func (r *Resources) ensureEncryptionKeyfolderSecret(ctx context.Context, cachedS } owner := r.context.GetAPIObject().AsOwner() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := AppendKeyfileToKeyfolder(ctxChild, cachedStatus, secrets, &owner, secretName, d) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return AppendKeyfileToKeyfolder(ctxChild, cachedStatus, secrets, &owner, secretName, d) + }) if err != nil { return errors.Wrapf(err, "Unable to create keyfolder secret") } @@ -562,9 +565,9 @@ func (r *Resources) ensureTLSCACertificateSecret(ctx context.Context, cachedStat apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := createTLSCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return createTLSCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) + }) if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil @@ -623,9 +626,9 @@ func (r *Resources) ensureClientAuthCACertificateSecret(ctx context.Context, cac apiObject := r.context.GetAPIObject() owner := apiObject.AsOwner() deploymentName := apiObject.GetName() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := createClientAuthCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return createClientAuthCACertificate(ctxChild, r.log, secrets, spec, deploymentName, &owner) + }) if k8sutil.IsAlreadyExists(err) { // Secret added while we tried it also return nil diff --git a/pkg/deployment/resources/servicemonitor.go b/pkg/deployment/resources/servicemonitor.go index aa9058298..7ab22965a 100644 --- a/pkg/deployment/resources/servicemonitor.go +++ b/pkg/deployment/resources/servicemonitor.go @@ -159,8 +159,8 @@ func (r *Resources) EnsureServiceMonitor(ctx context.Context) error { // Check if ServiceMonitor already exists serviceMonitors := mClient.ServiceMonitors(ns) ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() servMon, err := serviceMonitors.Get(ctxChild, serviceMonitorName, metav1.GetOptions{}) - cancel() if err != nil { if k8sutil.IsNotFound(err) { if !wantMetrics { @@ -182,9 +182,10 @@ func (r *Resources) EnsureServiceMonitor(ctx context.Context) error { Spec: spec, } - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - smon, err = serviceMonitors.Create(ctxChild, smon, metav1.CreateOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := serviceMonitors.Create(ctxChild, smon, metav1.CreateOptions{}) + return err + }) if err != nil { log.Error().Err(err).Msgf("Failed to create ServiceMonitor %s", serviceMonitorName) return errors.WithStack(err) @@ -226,9 +227,10 @@ func (r *Resources) EnsureServiceMonitor(ctx context.Context) error { servMon.Spec = spec - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err = serviceMonitors.Update(ctxChild, servMon, metav1.UpdateOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := serviceMonitors.Update(ctxChild, servMon, metav1.UpdateOptions{}) + return err + }) if err != nil { return err } @@ -236,9 +238,9 @@ func (r *Resources) EnsureServiceMonitor(ctx context.Context) error { return nil } // Need to get rid of the ServiceMonitor: - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err = serviceMonitors.Delete(ctxChild, serviceMonitorName, metav1.DeleteOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return serviceMonitors.Delete(ctxChild, serviceMonitorName, metav1.DeleteOptions{}) + }) if err == nil { log.Debug().Msgf("Deleted ServiceMonitor %s", serviceMonitorName) return nil diff --git a/pkg/deployment/resources/services.go b/pkg/deployment/resources/services.go index 420ba821c..c11e77149 100644 --- a/pkg/deployment/resources/services.go +++ b/pkg/deployment/resources/services.go @@ -100,9 +100,10 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn }, } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := svcs.Create(ctxChild, s, metav1.CreateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := svcs.Create(ctxChild, s, metav1.CreateOptions{}) + return err + }) if err != nil { if !k8sutil.IsConflict(err) { return err @@ -128,9 +129,10 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn if !equality.Semantic.DeepDerivative(*spec, s.Spec) { s.Spec = *spec - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := svcs.Update(ctxChild, s, metav1.UpdateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := svcs.Update(ctxChild, s, metav1.UpdateOptions{}) + return err + }) if err != nil { return err } @@ -149,8 +151,8 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn counterMetric.Inc() if _, exists := cachedStatus.Service(k8sutil.CreateHeadlessServiceName(deploymentName)); !exists { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() svcName, newlyCreated, err := k8sutil.CreateHeadlessService(ctxChild, svcs, apiObject, owner) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create headless service") return errors.WithStack(err) @@ -165,8 +167,8 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn counterMetric.Inc() if _, exists := cachedStatus.Service(k8sutil.CreateDatabaseClientServiceName(deploymentName)); !exists { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() svcName, newlyCreated, err := k8sutil.CreateDatabaseClientService(ctxChild, svcs, apiObject, single, owner) - cancel() if err != nil { log.Debug().Err(err).Msg("Failed to create database client service") return errors.WithStack(err) @@ -214,8 +216,8 @@ func (r *Resources) EnsureServices(ctx context.Context, cachedStatus inspectorIn if spec.Metrics.IsEnabled() { ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() name, _, err := k8sutil.CreateExporterService(ctxChild, cachedStatus, svcs, apiObject, apiObject.AsOwner()) - cancel() if err != nil { log.Debug().Err(err).Msgf("Failed to create %s exporter service", name) return errors.WithStack(err) @@ -288,9 +290,10 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat } } if updateExternalAccessService && !createExternalAccessService && !deleteExternalAccessService { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - _, err := svcs.Update(ctxChild, existing, metav1.UpdateOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := svcs.Update(ctxChild, existing, metav1.UpdateOptions{}) + return err + }) if err != nil { log.Debug().Err(err).Msgf("Failed to update %s external access service", title) return errors.WithStack(err) @@ -305,9 +308,9 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat if deleteExternalAccessService { log.Info().Str("service", eaServiceName).Msgf("Removing obsolete %s external access service", title) - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - err := svcs.Delete(ctxChild, eaServiceName, metav1.DeleteOptions{}) - cancel() + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + return svcs.Delete(ctxChild, eaServiceName, metav1.DeleteOptions{}) + }) if err != nil { log.Debug().Err(err).Msgf("Failed to remove %s external access service", title) return errors.WithStack(err) @@ -319,8 +322,8 @@ func (r *Resources) ensureExternalAccessServices(ctx context.Context, cachedStat loadBalancerIP := spec.GetLoadBalancerIP() loadBalancerSourceRanges := spec.LoadBalancerSourceRanges ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() _, newlyCreated, err := k8sutil.CreateExternalAccessService(ctxChild, svcs, eaServiceName, svcRole, apiObject, eaServiceType, port, nodePort, loadBalancerIP, loadBalancerSourceRanges, apiObject.AsOwner()) - cancel() if err != nil { log.Debug().Err(err).Msgf("Failed to create %s external access service", title) return errors.WithStack(err) diff --git a/pkg/util/arangod/client.go b/pkg/util/arangod/client.go index 47508db66..be179a9ab 100644 --- a/pkg/util/arangod/client.go +++ b/pkg/util/arangod/client.go @@ -50,6 +50,8 @@ type ( requireAuthenticationKey struct{} ) +type TimeoutRunFunc k8sutil.TimeoutRunFunc + const ( minArangoDDefaultTimeout = time.Second * 10 ) @@ -61,6 +63,16 @@ func GetRequestTimeout() time.Duration { return requestTimeout } +// RunWithTimeout runs the function with the provided timeout or with default timeout. +func RunWithTimeout(ctx context.Context, run TimeoutRunFunc, timeout ...time.Duration) error { + t := GetRequestTimeout() + if len(timeout) > 0 { + t = timeout[0] + } + + return k8sutil.RunWithTimeout(ctx, k8sutil.TimeoutRunFunc(run), t) +} + // SetRequestTimeout sets request timeout for one call to kubernetes. func SetRequestTimeout(timeout time.Duration) { if timeout > minArangoDDefaultTimeout { @@ -278,8 +290,8 @@ func createArangodClientAuthentication(ctx context.Context, cli corev1.CoreV1Int if ctx.Value(skipAuthenticationKey{}) == nil { secrets := cli.Secrets(apiObject.GetNamespace()) ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) + defer cancel() s, err := k8sutil.GetTokenSecret(ctxChild, secrets, apiObject.Spec.Authentication.GetJWTSecretName()) - cancel() if err != nil { return nil, errors.WithStack(err) } diff --git a/pkg/util/arangod/dbserver.go b/pkg/util/arangod/dbserver.go index 317d88443..974d6db1d 100644 --- a/pkg/util/arangod/dbserver.go +++ b/pkg/util/arangod/dbserver.go @@ -38,15 +38,15 @@ import ( // is not empty, or nil when the dbserver is found to be empty. func IsDBServerEmpty(ctx context.Context, id string, client driver.Client) error { ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() c, err := client.Cluster(ctxChild) - cancel() if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot obtain Cluster")) } ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() dbs, err := client.Databases(ctxChild) - cancel() if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot fetch databases")) } diff --git a/pkg/util/k8sutil/secrets.go b/pkg/util/k8sutil/secrets.go index 95386cc2c..de5b73171 100644 --- a/pkg/util/k8sutil/secrets.go +++ b/pkg/util/k8sutil/secrets.go @@ -302,8 +302,8 @@ func CreateJWTTokenFromSecret(secret string, claims map[string]interface{}) (str // result in a new secret called tokenSecretName func CreateJWTFromSecret(ctx context.Context, secrets SecretInterface, tokenSecretName, secretSecretName string, claims map[string]interface{}, ownerRef *meta.OwnerReference) error { ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) + defer cancel() secret, err := GetTokenSecret(ctxChild, secrets, secretSecretName) - cancel() if err != nil { return errors.WithStack(err) } @@ -317,9 +317,9 @@ func CreateJWTFromSecret(ctx context.Context, secrets SecretInterface, tokenSecr return errors.WithStack(err) } - ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) - defer cancel() - return CreateTokenSecret(ctxChild, secrets, tokenSecretName, signedToken, ownerRef) + return RunWithTimeout(ctx, func(ctxChild context.Context) error { + return CreateTokenSecret(ctxChild, secrets, tokenSecretName, signedToken, ownerRef) + }) } // CreateBasicAuthSecret creates a secret with given name in given namespace @@ -338,9 +338,11 @@ func CreateBasicAuthSecret(ctx context.Context, secrets SecretInterface, secretN } // Attach secret to owner AddOwnerRefToObject(secret, ownerRef) - ctxChild, cancel := context.WithTimeout(ctx, GetRequestTimeout()) - defer cancel() - if _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}); err != nil { + err := RunWithTimeout(ctx, func(ctxChild context.Context) error { + _, err := secrets.Create(ctxChild, secret, meta.CreateOptions{}) + return err + }) + if err != nil { // Failed to create secret return errors.WithStack(err) } diff --git a/pkg/util/k8sutil/util.go b/pkg/util/k8sutil/util.go index 7df03b01e..6ba1aa826 100644 --- a/pkg/util/k8sutil/util.go +++ b/pkg/util/k8sutil/util.go @@ -24,12 +24,15 @@ package k8sutil import ( + "context" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" ) +type TimeoutRunFunc func(ctxChild context.Context) error + const ( // LabelKeyArangoDeployment is the key of the label used to store the ArangoDeployment name in LabelKeyArangoDeployment = "arango_deployment" @@ -58,6 +61,19 @@ func GetRequestTimeout() time.Duration { return requestTimeout } +// RunWithTimeout runs the function with the provided timeout or with default timeout. +func RunWithTimeout(ctx context.Context, run TimeoutRunFunc, timeout ...time.Duration) error { + t := GetRequestTimeout() + if len(timeout) > 0 { + t = timeout[0] + } + + ctxChild, cancel := context.WithTimeout(ctx, t) + defer cancel() + + return run(ctxChild) +} + // SetRequestTimeout sets request timeout for one call to kubernetes. func SetRequestTimeout(timeout time.Duration) { if timeout > minDefaultRequestTimeout { From 84a04202a42ca56fccd2cf8ba9a84024726f5369 Mon Sep 17 00:00:00 2001 From: Tomasz Mielech Date: Fri, 23 Apr 2021 12:03:03 +0200 Subject: [PATCH 6/6] use defer cancel() everywhere --- pkg/deployment/deployment.go | 39 +++++++++++++------ pkg/deployment/deployment_inspector.go | 9 +++-- .../reconcile/plan_builder_tls_sni.go | 20 +++++++--- pkg/deployment/resources/pdbs.go | 9 +++-- pkg/util/arangod/dbserver.go | 10 +++-- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/pkg/deployment/deployment.go b/pkg/deployment/deployment.go index ce9d5dfe4..a0a3b5b2f 100644 --- a/pkg/deployment/deployment.go +++ b/pkg/deployment/deployment.go @@ -399,9 +399,14 @@ func (d *Deployment) updateCRStatus(ctx context.Context, force ...bool) error { if update.GetDeletionTimestamp() == nil { ensureFinalizers(update) } - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - newAPIObject, err := depls.Update(ctxChild, update, metav1.UpdateOptions{}) - cancel() + + var newAPIObject *api.ArangoDeployment + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + newAPIObject, err = depls.Update(ctxChild, update, metav1.UpdateOptions{}) + + return err + }) if err == nil { // Update internal object d.apiObject = newAPIObject @@ -412,9 +417,12 @@ func (d *Deployment) updateCRStatus(ctx context.Context, force ...bool) error { // Reload api object and try again var current *api.ArangoDeployment - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - current, err = depls.Get(ctxChild, update.GetName(), metav1.GetOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + current, err = depls.Get(ctxChild, update.GetName(), metav1.GetOptions{}) + + return err + }) if err == nil { update = current.DeepCopy() continue @@ -448,9 +456,13 @@ func (d *Deployment) updateCRSpec(ctx context.Context, newSpec api.DeploymentSpe update.Spec = newSpec update.Status = d.status.last ns := d.apiObject.GetNamespace() - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - newAPIObject, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Update(ctxChild, update, metav1.UpdateOptions{}) - cancel() + var newAPIObject *api.ArangoDeployment + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + newAPIObject, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Update(ctxChild, update, metav1.UpdateOptions{}) + + return err + }) if err == nil { // Update internal object d.apiObject = newAPIObject @@ -461,9 +473,12 @@ func (d *Deployment) updateCRSpec(ctx context.Context, newSpec api.DeploymentSpe // Reload api object and try again var current *api.ArangoDeployment - ctxChild, cancel = context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - current, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Get(ctxChild, update.GetName(), metav1.GetOptions{}) - cancel() + err = k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + current, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(ns).Get(ctxChild, update.GetName(), metav1.GetOptions{}) + + return err + }) if err == nil { update = current.DeepCopy() continue diff --git a/pkg/deployment/deployment_inspector.go b/pkg/deployment/deployment_inspector.go index 1c3000f11..f74be1e8a 100644 --- a/pkg/deployment/deployment_inspector.go +++ b/pkg/deployment/deployment_inspector.go @@ -109,9 +109,12 @@ func (d *Deployment) inspectDeployment(lastInterval util.Interval) util.Interval } // Check deployment still exists - ctx, cancel := context.WithTimeout(ctxReconciliation, k8sutil.GetRequestTimeout()) - updated, err := d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctx, deploymentName, metav1.GetOptions{}) - cancel() + var updated *api.ArangoDeployment + err = k8sutil.RunWithTimeout(ctxReconciliation, func(ctxChild context.Context) error { + var err error + updated, err = d.deps.DatabaseCRCli.DatabaseV1().ArangoDeployments(d.apiObject.GetNamespace()).Get(ctxChild, deploymentName, metav1.GetOptions{}) + return err + }) if k8sutil.IsNotFound(err) { // Deployment is gone log.Info().Msg("Deployment is gone") diff --git a/pkg/deployment/reconcile/plan_builder_tls_sni.go b/pkg/deployment/reconcile/plan_builder_tls_sni.go index 6b38a4350..cd59250c4 100644 --- a/pkg/deployment/reconcile/plan_builder_tls_sni.go +++ b/pkg/deployment/reconcile/plan_builder_tls_sni.go @@ -26,6 +26,8 @@ package reconcile import ( "context" + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/deployment/features" "github.com/arangodb/kube-arangodb/pkg/util/arangod" @@ -82,17 +84,23 @@ func createRotateTLSServerSNIPlan(ctx context.Context, continue } - ctxChild, cancel := context.WithTimeout(ctx, arangod.GetRequestTimeout()) - c, err := planCtx.GetServerClient(ctxChild, group, m.ID) - cancel() + var c driver.Client + err := arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + c, err = planCtx.GetServerClient(ctxChild, group, m.ID) + return err + }) if err != nil { log.Warn().Err(err).Msg("Unable to get client") continue } - ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout()) - ok, err := compareTLSSNIConfig(ctxChild, c.Connection(), fetchedSecrets, false) - cancel() + var ok bool + err = arangod.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + ok, err = compareTLSSNIConfig(ctxChild, c.Connection(), fetchedSecrets, false) + return err + }) if err != nil { log.Warn().Err(err).Msg("SNI compare failed") return nil diff --git a/pkg/deployment/resources/pdbs.go b/pkg/deployment/resources/pdbs.go index dc81dacec..b68cdf401 100644 --- a/pkg/deployment/resources/pdbs.go +++ b/pkg/deployment/resources/pdbs.go @@ -113,9 +113,12 @@ func (r *Resources) ensurePDBForGroup(ctx context.Context, group api.ServerGroup log := r.log.With().Str("group", group.AsRole()).Logger() for { - ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout()) - pdb, err := pdbcli.Get(ctxChild, pdbname, metav1.GetOptions{}) - cancel() + var pdb *policyv1beta1.PodDisruptionBudget + err := k8sutil.RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + pdb, err = pdbcli.Get(ctxChild, pdbname, metav1.GetOptions{}) + return err + }) if k8sutil.IsNotFound(err) { if wantedMinAvail != 0 { // No PDB found - create new diff --git a/pkg/util/arangod/dbserver.go b/pkg/util/arangod/dbserver.go index 974d6db1d..559ed4932 100644 --- a/pkg/util/arangod/dbserver.go +++ b/pkg/util/arangod/dbserver.go @@ -51,10 +51,14 @@ func IsDBServerEmpty(ctx context.Context, id string, client driver.Client) error return errors.WithStack(errors.Wrapf(err, "Cannot fetch databases")) } + var inventory driver.DatabaseInventory for _, db := range dbs { - ctxChild, cancel = context.WithTimeout(ctx, GetRequestTimeout()) - inventory, err := c.DatabaseInventory(ctxChild, db) - cancel() + err := RunWithTimeout(ctx, func(ctxChild context.Context) error { + var err error + inventory, err = c.DatabaseInventory(ctxChild, db) + + return err + }) if err != nil { return errors.WithStack(errors.Wrapf(err, "Cannot fetch inventory for %s", db.Name())) }