From 599bac90f239e1eb62fc3a6050662518a97e16cc Mon Sep 17 00:00:00 2001 From: jwierzbo Date: Wed, 8 Mar 2023 10:23:43 +0100 Subject: [PATCH 1/7] GT-143 ArangoBackup create retries and MaxIterations limit --- CHANGELOG.md | 1 + pkg/apis/backup/v1/backup_spec_backoff.go | 2 ++ pkg/apis/backup/v1/backup_state.go | 1 + pkg/apis/backup/v1/backup_status_backoff.go | 8 ++++++++ pkg/handlers/backup/state.go | 1 + pkg/handlers/backup/state_create.go | 20 +++++++++++++++++++- 6 files changed, 32 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e876db316..71a9276e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ - (Maintenance) Generate README Platforms - (Improvement) Cleanout calculation - picks members with the lowest number of shards - (Improvement) Add new field to CR for more precise calculation of DC2DC replication progress +- (Feature) ArangoBackup create retries and MaxIterations limit ## [1.2.24](https://github.com/arangodb/kube-arangodb/tree/1.2.24) (2023-01-25) - (Bugfix) Fix deployment creation on ARM64 diff --git a/pkg/apis/backup/v1/backup_spec_backoff.go b/pkg/apis/backup/v1/backup_spec_backoff.go index ebbdd4600..15269c4c2 100644 --- a/pkg/apis/backup/v1/backup_spec_backoff.go +++ b/pkg/apis/backup/v1/backup_spec_backoff.go @@ -29,6 +29,8 @@ type ArangoBackupSpecBackOff struct { MaxDelay *int `json:"max_delay,omitempty"` // Iterations defines number of iterations before reaching MaxDelay. Default to 5 Iterations *int `json:"iterations,omitempty"` + // MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil + MaxIterations *int `json:"max_iterations,omitempty"` } func (a *ArangoBackupSpecBackOff) GetMaxDelay() int { diff --git a/pkg/apis/backup/v1/backup_state.go b/pkg/apis/backup/v1/backup_state.go index d990ab4e7..148e35ba5 100644 --- a/pkg/apis/backup/v1/backup_state.go +++ b/pkg/apis/backup/v1/backup_state.go @@ -34,6 +34,7 @@ const ( ArangoBackupStateDownloadError state.State = "DownloadError" ArangoBackupStateDownloading state.State = "Downloading" ArangoBackupStateCreate state.State = "Create" + ArangoBackupStateCreateError state.State = "CreateError" ArangoBackupStateUpload state.State = "Upload" ArangoBackupStateUploading state.State = "Uploading" ArangoBackupStateUploadError state.State = "UploadError" diff --git a/pkg/apis/backup/v1/backup_status_backoff.go b/pkg/apis/backup/v1/backup_status_backoff.go index 96c77edd3..cdbb26e94 100644 --- a/pkg/apis/backup/v1/backup_status_backoff.go +++ b/pkg/apis/backup/v1/backup_status_backoff.go @@ -52,6 +52,14 @@ func (a *ArangoBackupStatusBackOff) GetNext() meta.Time { } func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *ArangoBackupStatusBackOff { + if spec.MaxIterations != nil && a.GetIterations() >= *spec.MaxIterations { + // Do not backoff anymore + return &ArangoBackupStatusBackOff{ + Iterations: a.Iterations, + Next: a.Next, + } + } + return &ArangoBackupStatusBackOff{ Iterations: a.GetIterations() + 1, Next: meta.Time{Time: time.Now().Add(spec.Backoff(a.GetIterations()))}, diff --git a/pkg/handlers/backup/state.go b/pkg/handlers/backup/state.go index 6315b5010..523725fe0 100644 --- a/pkg/handlers/backup/state.go +++ b/pkg/handlers/backup/state.go @@ -33,6 +33,7 @@ var ( backupApi.ArangoBackupStatePending: statePendingHandler, backupApi.ArangoBackupStateScheduled: stateScheduledHandler, backupApi.ArangoBackupStateCreate: stateCreateHandler, + backupApi.ArangoBackupStateCreateError: stateCreateErrorHandler, backupApi.ArangoBackupStateUpload: stateUploadHandler, backupApi.ArangoBackupStateUploading: stateUploadingHandler, backupApi.ArangoBackupStateUploadError: stateUploadErrorHandler, diff --git a/pkg/handlers/backup/state_create.go b/pkg/handlers/backup/state_create.go index 709b275c4..f361544e6 100644 --- a/pkg/handlers/backup/state_create.go +++ b/pkg/handlers/backup/state_create.go @@ -21,6 +21,8 @@ package backup import ( + "time" + "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1" @@ -52,12 +54,28 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. ) } - return nil, newFatalError(err) + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateCreateError, "Create failed with error: %s", err.Error()), + cleanStatusJob(), + updateStatusAvailable(false), + addBackOff(backup.Spec), + ) } return wrapUpdateStatus(backup, updateStatusState(backupApi.ArangoBackupStateReady, ""), updateStatusAvailable(true), updateStatusBackup(backupMeta), + cleanBackOff(), ) } + +func stateCreateErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + if !backup.Status.Backoff.GetNext().After(time.Now()) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateCreate, ""), + cleanStatusJob()) + } + + return wrapUpdateStatus(backup) +} From c844207f070b679b1d2c088623563be9e1bda42c Mon Sep 17 00:00:00 2001 From: jwierzbo Date: Wed, 8 Mar 2023 11:30:53 +0100 Subject: [PATCH 2/7] MaxIterations logic --- pkg/apis/backup/v1/backup_spec_backoff.go | 2 +- pkg/apis/backup/v1/backup_status_backoff.go | 10 +++++++--- pkg/handlers/backup/state_create.go | 2 +- pkg/handlers/backup/state_uploaderror.go | 3 ++- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/apis/backup/v1/backup_spec_backoff.go b/pkg/apis/backup/v1/backup_spec_backoff.go index 15269c4c2..6517a048c 100644 --- a/pkg/apis/backup/v1/backup_spec_backoff.go +++ b/pkg/apis/backup/v1/backup_spec_backoff.go @@ -29,7 +29,7 @@ type ArangoBackupSpecBackOff struct { MaxDelay *int `json:"max_delay,omitempty"` // Iterations defines number of iterations before reaching MaxDelay. Default to 5 Iterations *int `json:"iterations,omitempty"` - // MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil + // MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil (no limit) MaxIterations *int `json:"max_iterations,omitempty"` } diff --git a/pkg/apis/backup/v1/backup_status_backoff.go b/pkg/apis/backup/v1/backup_status_backoff.go index cdbb26e94..195366222 100644 --- a/pkg/apis/backup/v1/backup_status_backoff.go +++ b/pkg/apis/backup/v1/backup_status_backoff.go @@ -51,12 +51,16 @@ func (a *ArangoBackupStatusBackOff) GetNext() meta.Time { return a.Next } +func (a *ArangoBackupStatusBackOff) ShouldBackoff(spec *ArangoBackupSpecBackOff) bool { + return spec == nil || spec.MaxIterations == nil || a.GetIterations() < *spec.MaxIterations +} + func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *ArangoBackupStatusBackOff { - if spec.MaxIterations != nil && a.GetIterations() >= *spec.MaxIterations { + if !a.ShouldBackoff(spec) { // Do not backoff anymore return &ArangoBackupStatusBackOff{ - Iterations: a.Iterations, - Next: a.Next, + Iterations: a.GetIterations(), + Next: a.GetNext(), } } diff --git a/pkg/handlers/backup/state_create.go b/pkg/handlers/backup/state_create.go index f361544e6..bac90e20d 100644 --- a/pkg/handlers/backup/state_create.go +++ b/pkg/handlers/backup/state_create.go @@ -71,7 +71,7 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. } func stateCreateErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { - if !backup.Status.Backoff.GetNext().After(time.Now()) { + if backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now()) { return wrapUpdateStatus(backup, updateStatusState(backupApi.ArangoBackupStateCreate, ""), cleanStatusJob()) diff --git a/pkg/handlers/backup/state_uploaderror.go b/pkg/handlers/backup/state_uploaderror.go index a1deaa635..ffdcd97ca 100644 --- a/pkg/handlers/backup/state_uploaderror.go +++ b/pkg/handlers/backup/state_uploaderror.go @@ -27,7 +27,8 @@ import ( ) func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { - if backup.Spec.Upload == nil || !backup.Status.Backoff.GetNext().After(time.Now()) { + if backup.Spec.Upload == nil || + (backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now())) { return wrapUpdateStatus(backup, updateStatusState(backupApi.ArangoBackupStateReady, ""), cleanStatusJob(), From 88463248a2e18d051bab1e72a4d525aca81793c4 Mon Sep 17 00:00:00 2001 From: jwierzbo Date: Wed, 8 Mar 2023 13:26:40 +0100 Subject: [PATCH 3/7] fix --- pkg/apis/backup/v1/backup_state.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/apis/backup/v1/backup_state.go b/pkg/apis/backup/v1/backup_state.go index 148e35ba5..ea70736cf 100644 --- a/pkg/apis/backup/v1/backup_state.go +++ b/pkg/apis/backup/v1/backup_state.go @@ -51,7 +51,8 @@ var ArangoBackupStateMap = state.Map{ ArangoBackupStateDownload: {ArangoBackupStateDownloading, ArangoBackupStateFailed, ArangoBackupStateDownloadError}, ArangoBackupStateDownloading: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateDownloadError}, ArangoBackupStateDownloadError: {ArangoBackupStatePending, ArangoBackupStateFailed}, - ArangoBackupStateCreate: {ArangoBackupStateReady, ArangoBackupStateFailed}, + ArangoBackupStateCreate: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateCreateError}, + ArangoBackupStateCreateError: {ArangoBackupStateFailed, ArangoBackupStateCreate}, ArangoBackupStateUpload: {ArangoBackupStateUploading, ArangoBackupStateFailed, ArangoBackupStateDeleted, ArangoBackupStateUploadError}, ArangoBackupStateUploading: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateUploadError}, ArangoBackupStateUploadError: {ArangoBackupStateFailed, ArangoBackupStateReady}, From 179d9407a5066642e742bc1bea6d1da4fc1db51a Mon Sep 17 00:00:00 2001 From: jwierzbo Date: Wed, 8 Mar 2023 15:19:28 +0100 Subject: [PATCH 4/7] UT --- .../backup/v1/backup_status_backoff_test.go | 19 +++++++++ pkg/apis/backup/v1/zz_generated.deepcopy.go | 5 +++ .../versioned/fake/clientset_generated.go | 5 +-- pkg/handlers/backup/state_create.go | 14 +++---- pkg/handlers/backup/state_create_test.go | 40 +++++++++++-------- 5 files changed, 56 insertions(+), 27 deletions(-) diff --git a/pkg/apis/backup/v1/backup_status_backoff_test.go b/pkg/apis/backup/v1/backup_status_backoff_test.go index 8ca9a4b0f..2707ab4ad 100644 --- a/pkg/apis/backup/v1/backup_status_backoff_test.go +++ b/pkg/apis/backup/v1/backup_status_backoff_test.go @@ -24,6 +24,8 @@ import ( "testing" "time" + "github.com/arangodb/kube-arangodb/pkg/util" + "github.com/stretchr/testify/require" ) @@ -37,4 +39,21 @@ func TestArangoBackupStatusBackOff_Backoff(t *testing.T) { require.Equal(t, 1, n.GetIterations()) require.True(t, n.GetNext().After(time.Now().Add(time.Duration(9.9*float64(time.Second))))) }) + + t.Run("Test MaxIterations", func(t *testing.T) { + var spec = &ArangoBackupSpecBackOff{ + Iterations: util.NewInt(2), + MaxIterations: util.NewInt(3), + } + var status *ArangoBackupStatusBackOff + + n := status.Backoff(spec) + require.Equal(t, 1, n.GetIterations()) + require.True(t, n.ShouldBackoff(spec)) + + n.Iterations = 3 + n = n.Backoff(spec) + require.Equal(t, 3, n.GetIterations()) + require.False(t, n.ShouldBackoff(spec)) + }) } diff --git a/pkg/apis/backup/v1/zz_generated.deepcopy.go b/pkg/apis/backup/v1/zz_generated.deepcopy.go index adc315c1a..a34052661 100644 --- a/pkg/apis/backup/v1/zz_generated.deepcopy.go +++ b/pkg/apis/backup/v1/zz_generated.deepcopy.go @@ -310,6 +310,11 @@ func (in *ArangoBackupSpecBackOff) DeepCopyInto(out *ArangoBackupSpecBackOff) { *out = new(int) **out = **in } + if in.MaxIterations != nil { + in, out := &in.MaxIterations, &out.MaxIterations + *out = new(int) + **out = **in + } return } diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index aeecfc040..7bdc53180 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -90,10 +90,7 @@ func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } -var ( - _ clientset.Interface = &Clientset{} - _ testing.FakeClient = &Clientset{} -) +var _ clientset.Interface = &Clientset{} // AppsV1 retrieves the AppsV1Client func (c *Clientset) AppsV1() appsv1.AppsV1Interface { diff --git a/pkg/handlers/backup/state_create.go b/pkg/handlers/backup/state_create.go index bac90e20d..6952c11ab 100644 --- a/pkg/handlers/backup/state_create.go +++ b/pkg/handlers/backup/state_create.go @@ -41,7 +41,12 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. response, err := client.Create() if err != nil { - return nil, err + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateCreateError, "Create failed with error: %s", err.Error()), + cleanStatusJob(), + updateStatusAvailable(false), + addBackOff(backup.Spec), + ) } backupMeta, err := client.Get(response.ID) @@ -54,12 +59,7 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. ) } - return wrapUpdateStatus(backup, - updateStatusState(backupApi.ArangoBackupStateCreateError, "Create failed with error: %s", err.Error()), - cleanStatusJob(), - updateStatusAvailable(false), - addBackOff(backup.Spec), - ) + return nil, newFatalError(err) } return wrapUpdateStatus(backup, diff --git a/pkg/handlers/backup/state_create_test.go b/pkg/handlers/backup/state_create_test.go index d7108bec3..bca80a0cc 100644 --- a/pkg/handlers/backup/state_create_test.go +++ b/pkg/handlers/backup/state_create_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,8 +22,10 @@ package backup import ( "testing" + "time" "github.com/stretchr/testify/require" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/arangodb/go-driver" @@ -120,7 +122,7 @@ func Test_State_Create_Upload(t *testing.T) { compareBackupMeta(t, backupMeta, newObj) } -func Test_State_Create_CreateFailed(t *testing.T) { +func Test_State_Create_CreateError(t *testing.T) { // Arrange error := newFatalErrorf("error") handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ @@ -137,32 +139,38 @@ func Test_State_Create_CreateFailed(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) - require.Equal(t, newObj.Status.Message, createStateMessage(backupApi.ArangoBackupStateCreate, backupApi.ArangoBackupStateFailed, error.Error())) - + require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreateError) require.Nil(t, newObj.Status.Backup) - require.False(t, newObj.Status.Available) } -func Test_State_Create_TemporaryCreateFailed(t *testing.T) { +func Test_State_CreateError_Retry(t *testing.T) { // Arrange - error := newTemporaryErrorf("error") - handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - createError: error, - }) + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) - obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreate) + obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreateError) + + backupMeta, err := mock.Create() + require.NoError(t, err) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{ + ID: string(backupMeta.ID), + Version: backupMeta.Version, + CreationTimestamp: meta.Now(), + } + + obj.Status.Time.Time = time.Now().Add(-2 * downloadDelay) // Act createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - err := handler.Handle(newItemFromBackup(operation.Update, obj)) - require.EqualError(t, err, error.Error()) + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) // Assert newObj := refreshArangoBackup(t, handler, obj) - - require.Equal(t, obj.Status, newObj.Status) + require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreate) + require.False(t, newObj.Status.Available) + require.NotNil(t, newObj.Status.Backup) + require.Equal(t, obj.Status.Backup, newObj.Status.Backup) } From 483f13405c7062e237cc39166065f20d99fc9b3d Mon Sep 17 00:00:00 2001 From: jwierzbo Date: Wed, 8 Mar 2023 16:09:49 +0100 Subject: [PATCH 5/7] fix --- pkg/apis/backup/v1/backup_status_backoff_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/apis/backup/v1/backup_status_backoff_test.go b/pkg/apis/backup/v1/backup_status_backoff_test.go index 2707ab4ad..5b2577868 100644 --- a/pkg/apis/backup/v1/backup_status_backoff_test.go +++ b/pkg/apis/backup/v1/backup_status_backoff_test.go @@ -24,9 +24,9 @@ import ( "testing" "time" - "github.com/arangodb/kube-arangodb/pkg/util" - "github.com/stretchr/testify/require" + + "github.com/arangodb/kube-arangodb/pkg/util" ) func TestArangoBackupStatusBackOff_Backoff(t *testing.T) { From 29528513c40f4f32adea855f1b405e6fa7921e7f Mon Sep 17 00:00:00 2001 From: jwierzbo Date: Thu, 9 Mar 2023 16:41:04 +0100 Subject: [PATCH 6/7] Trnsfer to Failed state after reaching MaxRetries --- pkg/handlers/backup/state_create.go | 9 ++++++ pkg/handlers/backup/state_create_test.go | 37 ++++++++++++++++++++++++ pkg/handlers/backup/state_uploaderror.go | 9 ++++++ 3 files changed, 55 insertions(+) diff --git a/pkg/handlers/backup/state_create.go b/pkg/handlers/backup/state_create.go index 6952c11ab..54971624e 100644 --- a/pkg/handlers/backup/state_create.go +++ b/pkg/handlers/backup/state_create.go @@ -71,11 +71,20 @@ func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi. } func stateCreateErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + // no more retries - move to failed state + if !backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateFailed, "out of Create retries"), + cleanStatusJob()) + } + + // if we should retry - move to create state if backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now()) { return wrapUpdateStatus(backup, updateStatusState(backupApi.ArangoBackupStateCreate, ""), cleanStatusJob()) } + // no ready to retry - wait (do not change state) return wrapUpdateStatus(backup) } diff --git a/pkg/handlers/backup/state_create_test.go b/pkg/handlers/backup/state_create_test.go index bca80a0cc..b6d9eafd5 100644 --- a/pkg/handlers/backup/state_create_test.go +++ b/pkg/handlers/backup/state_create_test.go @@ -174,3 +174,40 @@ func Test_State_CreateError_Retry(t *testing.T) { require.NotNil(t, newObj.Status.Backup) require.Equal(t, obj.Status.Backup, newObj.Status.Backup) } + +func Test_State_CreateError_Transfer_To_Failed(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreateError) + + backupMeta, err := mock.Create() + require.NoError(t, err) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{ + ID: string(backupMeta.ID), + Version: backupMeta.Version, + CreationTimestamp: meta.Now(), + } + obj.Status.Backoff = &backupApi.ArangoBackupStatusBackOff{ + Iterations: 2, + } + + obj.Spec.Backoff = &backupApi.ArangoBackupSpecBackOff{ + Iterations: util.NewInt(1), + MaxIterations: util.NewInt(2), + } + + obj.Status.Time.Time = time.Now().Add(-2 * downloadDelay) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) + require.Equal(t, newObj.Status.Message, "out of Create retries") +} diff --git a/pkg/handlers/backup/state_uploaderror.go b/pkg/handlers/backup/state_uploaderror.go index ffdcd97ca..a51e6951a 100644 --- a/pkg/handlers/backup/state_uploaderror.go +++ b/pkg/handlers/backup/state_uploaderror.go @@ -27,6 +27,14 @@ import ( ) func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + // no more retries - move to failed state + if !backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateFailed, "out of Upload retries"), + cleanStatusJob()) + } + + // if we should retry - move to ready state if backup.Spec.Upload == nil || (backup.Status.Backoff.ShouldBackoff(backup.Spec.Backoff) && !backup.Status.Backoff.GetNext().After(time.Now())) { return wrapUpdateStatus(backup, @@ -35,6 +43,7 @@ func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backu updateStatusAvailable(true)) } + // no ready to retry - wait (do not change state) return wrapUpdateStatus(backup, updateStatusAvailable(true)) } From c6ff808df347ec6be641c3775d28642f42a87b9f Mon Sep 17 00:00:00 2001 From: ajanikow <12255597+ajanikow@users.noreply.github.com> Date: Tue, 2 May 2023 09:04:49 +0000 Subject: [PATCH 7/7] Fix header --- pkg/apis/backup/v1/backup_spec_backoff.go | 2 +- pkg/apis/backup/v1/backup_state.go | 2 +- pkg/apis/backup/v1/backup_status_backoff.go | 2 +- pkg/apis/backup/v1/backup_status_backoff_test.go | 2 +- .../clientset/versioned/fake/clientset_generated.go | 5 ++++- pkg/handlers/backup/state.go | 2 +- pkg/handlers/backup/state_create.go | 2 +- pkg/handlers/backup/state_uploaderror.go | 2 +- 8 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pkg/apis/backup/v1/backup_spec_backoff.go b/pkg/apis/backup/v1/backup_spec_backoff.go index 6517a048c..fbd8ecdfb 100644 --- a/pkg/apis/backup/v1/backup_spec_backoff.go +++ b/pkg/apis/backup/v1/backup_spec_backoff.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/apis/backup/v1/backup_state.go b/pkg/apis/backup/v1/backup_state.go index ea70736cf..ecfcf30a4 100644 --- a/pkg/apis/backup/v1/backup_state.go +++ b/pkg/apis/backup/v1/backup_state.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/apis/backup/v1/backup_status_backoff.go b/pkg/apis/backup/v1/backup_status_backoff.go index 195366222..6e2df565f 100644 --- a/pkg/apis/backup/v1/backup_status_backoff.go +++ b/pkg/apis/backup/v1/backup_status_backoff.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/apis/backup/v1/backup_status_backoff_test.go b/pkg/apis/backup/v1/backup_status_backoff_test.go index 5b2577868..9c8d13d77 100644 --- a/pkg/apis/backup/v1/backup_status_backoff_test.go +++ b/pkg/apis/backup/v1/backup_status_backoff_test.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/generated/clientset/versioned/fake/clientset_generated.go b/pkg/generated/clientset/versioned/fake/clientset_generated.go index 7bdc53180..aeecfc040 100644 --- a/pkg/generated/clientset/versioned/fake/clientset_generated.go +++ b/pkg/generated/clientset/versioned/fake/clientset_generated.go @@ -90,7 +90,10 @@ func (c *Clientset) Tracker() testing.ObjectTracker { return c.tracker } -var _ clientset.Interface = &Clientset{} +var ( + _ clientset.Interface = &Clientset{} + _ testing.FakeClient = &Clientset{} +) // AppsV1 retrieves the AppsV1Client func (c *Clientset) AppsV1() appsv1.AppsV1Interface { diff --git a/pkg/handlers/backup/state.go b/pkg/handlers/backup/state.go index 523725fe0..979c254d3 100644 --- a/pkg/handlers/backup/state.go +++ b/pkg/handlers/backup/state.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/handlers/backup/state_create.go b/pkg/handlers/backup/state_create.go index 54971624e..b75d2136c 100644 --- a/pkg/handlers/backup/state_create.go +++ b/pkg/handlers/backup/state_create.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/handlers/backup/state_uploaderror.go b/pkg/handlers/backup/state_uploaderror.go index a51e6951a..abfa4edc5 100644 --- a/pkg/handlers/backup/state_uploaderror.go +++ b/pkg/handlers/backup/state_uploaderror.go @@ -1,7 +1,7 @@ // // DISCLAIMER // -// Copyright 2016-2022 ArangoDB GmbH, Cologne, Germany +// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License.