diff --git a/chart/kube-arangodb-crd/templates/backup-policy.yaml b/chart/kube-arangodb-crd/templates/backup-policy.yaml index fdeb42f16..cd6cb559a 100644 --- a/chart/kube-arangodb-crd/templates/backup-policy.yaml +++ b/chart/kube-arangodb-crd/templates/backup-policy.yaml @@ -1,7 +1,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: arangobackups.backup.arangodb.com + name: arangobackuppolicies.backup.arangodb.com labels: app.kubernetes.io/name: {{ template "kube-arangodb-crd.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} @@ -11,39 +11,28 @@ metadata: spec: group: backup.arangodb.com additionalPrinterColumns: - - JSONPath: .spec.policyName - description: Policy name - name: Policy + - JSONPath: .spec.schedule + description: Schedule + name: Schedule type: string - - JSONPath: .spec.deployment.name - description: Deployment name - name: Deployment - type: string - - JSONPath: .status.backup.version - description: Backup Version - name: Version - type: string - - JSONPath: .status.backup.createdAt - description: Backup Creation Timestamp - name: Created - type: string - - JSONPath: .status.state - description: The actual state of the ArangoBackup - name: State + - JSONPath: .status.scheduled + description: Scheduled + name: Scheduled type: string - JSONPath: .status.message priority: 1 - description: Message of the ArangoBackup object + description: Message of the ArangoBackupPolicy object name: Message type: string names: - kind: ArangoBackup - listKind: ArangoBackupList - plural: arangobackups + kind: ArangoBackupPolicy + listKind: ArangoBackupPolicyList + plural: arangobackuppolicies shortNames: - - arangobackup - singular: arangobackup + - arangobackuppolicy + - arangobp + singular: arangobackuppolicy scope: Namespaced version: v1alpha subresources: - status: {} \ No newline at end of file + status: {} diff --git a/chart/kube-arangodb-crd/templates/backup.yaml b/chart/kube-arangodb-crd/templates/backup.yaml index cd6cb559a..2210da01d 100644 --- a/chart/kube-arangodb-crd/templates/backup.yaml +++ b/chart/kube-arangodb-crd/templates/backup.yaml @@ -1,7 +1,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: arangobackuppolicies.backup.arangodb.com + name: arangobackups.backup.arangodb.com labels: app.kubernetes.io/name: {{ template "kube-arangodb-crd.name" . }} helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }} @@ -11,28 +11,48 @@ metadata: spec: group: backup.arangodb.com additionalPrinterColumns: - - JSONPath: .spec.schedule - description: Schedule - name: Schedule + - JSONPath: .spec.policyName + description: Policy name + name: Policy type: string - - JSONPath: .status.scheduled - description: Scheduled - name: Scheduled + - JSONPath: .spec.deployment.name + description: Deployment name + name: Deployment + type: string + - JSONPath: .status.backup.version + description: Backup Version + name: Version + type: string + - JSONPath: .status.backup.createdAt + description: Backup Creation Timestamp + name: Created + type: string + - JSONPath: .status.backup.sizeInBytes + description: Backup Size in Bytes + name: Size + type: integer + format: byte + - JSONPath: .status.backup.numberOfDBServers + description: Backup Number of the DB Servers + name: DBServers + type: integer + - JSONPath: .status.state + description: The actual state of the ArangoBackup + name: State type: string - JSONPath: .status.message priority: 1 - description: Message of the ArangoBackupPolicy object + description: Message of the ArangoBackup object name: Message type: string names: - kind: ArangoBackupPolicy - listKind: ArangoBackupPolicyList - plural: arangobackuppolicies + kind: ArangoBackup + listKind: ArangoBackupList + plural: arangobackups shortNames: - - arangobackuppolicy - - arangobp - singular: arangobackuppolicy + - arangobackup + singular: arangobackup scope: Namespaced version: v1alpha subresources: - status: {} + status: {} \ No newline at end of file diff --git a/chart/kube-arangodb/templates/backup-operator/role.yaml b/chart/kube-arangodb/templates/backup-operator/role.yaml index c99686d44..8d4aff472 100644 --- a/chart/kube-arangodb/templates/backup-operator/role.yaml +++ b/chart/kube-arangodb/templates/backup-operator/role.yaml @@ -16,6 +16,9 @@ rules: - apiGroups: [""] resources: ["pods", "services", "endpoints"] verbs: ["get", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["*"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] diff --git a/docs/Manual/Deployment/Kubernetes/BackupResource.md b/docs/Manual/Deployment/Kubernetes/BackupResource.md index aeb6755e0..1b0e1cc1e 100644 --- a/docs/Manual/Deployment/Kubernetes/BackupResource.md +++ b/docs/Manual/Deployment/Kubernetes/BackupResource.md @@ -114,6 +114,8 @@ status: uploaded: true downloaded: true createdAt: "time" + sizeInBytes: 1 + numberOfDBServers: 3 available: true ``` @@ -331,6 +333,7 @@ Possible states: - "Ready" - state when Backup is finished - "Deleted" - state when Backup was once in ready, but has been deleted - "Failed" - state for failure +- "Unavailable" - state when Backup is not available on the ArangoDB. It can happen in case of upgrades, node restarts etc. ### `status.time: timestamp` @@ -429,6 +432,22 @@ Required: true Default: now() +#### `status.backup.sizeInBytes: uint64` + +Size of the Backup in ArangoDB. + +Required: true + +Default: 0 + +#### `status.backup.numberOfDBServers: uint` + +Cluster size of the Backup in ArangoDB. + +Required: true + +Default: 0 + ### `status.available: bool` Determines if we can restore from ArangoBackup. diff --git a/go.mod b/go.mod index 5d620200b..41695ed25 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/aktau/github-release v0.7.2 // indirect github.com/arangodb-helper/go-certificates v0.0.0-20180821055445-9fca24fc2680 github.com/arangodb/arangosync-client v0.6.3 - github.com/arangodb/go-driver v0.0.0-20190917135832-4ec315b17bf0 + github.com/arangodb/go-driver v0.0.0-20191002124627-11b6bfc64f67 github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect github.com/bugagazavr/go-gitlab-client v0.0.0-20150830002541-e5999f934dc4 // indirect @@ -62,6 +62,7 @@ require ( github.com/juju/errgo v0.0.0-20140925100237-08cceb5d0b53 // indirect github.com/julienschmidt/httprouter v1.2.0 github.com/manucorporat/stats v0.0.0-20180402194714-3ba42d56d227 // indirect + github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a // indirect github.com/mattn/go-colorable v0.1.1 // indirect github.com/mattn/go-isatty v0.0.7 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect diff --git a/go.sum b/go.sum index 95df647d3..b623f6082 100644 --- a/go.sum +++ b/go.sum @@ -71,6 +71,12 @@ github.com/arangodb/go-driver v0.0.0-20190806145426-48dd51aa0940 h1:CWLm7jOfT3v3 github.com/arangodb/go-driver v0.0.0-20190806145426-48dd51aa0940/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w= github.com/arangodb/go-driver v0.0.0-20190917135832-4ec315b17bf0 h1:OZAHgwP/X6eJlm33G94lo32j82acWZZvB2d/hbSAbTs= github.com/arangodb/go-driver v0.0.0-20190917135832-4ec315b17bf0/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w= +github.com/arangodb/go-driver v0.0.0-20191001090752-e07a8fe34b53 h1:gKOy8ZeLmAMWoCfZtXAwTpl9S3UxHjv8l4T96BvuXoQ= +github.com/arangodb/go-driver v0.0.0-20191001090752-e07a8fe34b53/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w= +github.com/arangodb/go-driver v0.0.0-20191002081852-02f5e9c3a694 h1:Cn68e0b50vTJqku80erSNWddXixyyvzVOy7cgb4MKYw= +github.com/arangodb/go-driver v0.0.0-20191002081852-02f5e9c3a694/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w= +github.com/arangodb/go-driver v0.0.0-20191002124627-11b6bfc64f67 h1:Nmcpj1EUYYOs3x/4F/4GH2nA8ooxkInWjOmbyOlw4bQ= +github.com/arangodb/go-driver v0.0.0-20191002124627-11b6bfc64f67/go.mod h1:5nMxlcbN6PHA05uGUw2EH6PIQHvdRVTrg/S/GM8jG4w= github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21 h1:+W7D5ttxi/Ygh/39vialtypE23p9KI7P0J2qtoqUV4w= github.com/arangodb/go-upgrade-rules v0.0.0-20180809110947-031b4774ff21/go.mod h1:RkPIG6JJ2pcJUoymc18NxAJGraZd+iAEVnOTDjZey/w= github.com/arangodb/go-velocypack v0.0.0-20190129082528-7896a965b4ad h1:Ah+VRYUWLuqgbfnDyuC8IrIe8XFzpt9tBVVnPGFNTJ8= @@ -414,6 +420,8 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe h1:W/GaMY0y69G4cFl github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/manucorporat/stats v0.0.0-20180402194714-3ba42d56d227 h1:KIaAZ/V+/0/6BOULrmBQ9T1ed8BkKqGIjIKW923nJuo= github.com/manucorporat/stats v0.0.0-20180402194714-3ba42d56d227/go.mod h1:ruMr5t05gVho4tuDv0PbI0Bb8nOxc/5Y6JzRHe/yfA0= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a h1:+J2gw7Bw77w/fbK7wnNJJDKmw1IbWft2Ul5BzrG1Qm8= +github.com/mattbaird/jsonpatch v0.0.0-20171005235357-81af80346b1a/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= diff --git a/manifests/arango-backup.yaml b/manifests/arango-backup.yaml index 7c7bbdb7f..e7b27daa5 100644 --- a/manifests/arango-backup.yaml +++ b/manifests/arango-backup.yaml @@ -66,6 +66,9 @@ rules: - apiGroups: [""] resources: ["pods", "services", "endpoints"] verbs: ["get", "update"] + - apiGroups: [""] + resources: ["events"] + verbs: ["*"] - apiGroups: [""] resources: ["secrets"] verbs: ["get"] @@ -168,6 +171,9 @@ spec: operator: In values: - amd64 + hostNetwork: false + hostPID: false + hostIPC: false containers: - name: operator imagePullPolicy: Always @@ -193,9 +199,12 @@ spec: - name: metrics containerPort: 8528 securityContext: - capabilities: - drop: - - 'ALL' + privileged: false + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - 'ALL' livenessProbe: httpGet: path: /health diff --git a/manifests/arango-crd.yaml b/manifests/arango-crd.yaml index 23ed0adb9..495759439 100644 --- a/manifests/arango-crd.yaml +++ b/manifests/arango-crd.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: arangobackups.backup.arangodb.com + name: arangobackuppolicies.backup.arangodb.com labels: app.kubernetes.io/name: kube-arangodb-crd helm.sh/chart: kube-arangodb-crd-0.3.16 @@ -13,48 +13,38 @@ metadata: spec: group: backup.arangodb.com additionalPrinterColumns: - - JSONPath: .spec.policyName - description: Policy name - name: Policy - type: string - - JSONPath: .spec.deployment.name - description: Deployment name - name: Deployment - type: string - - JSONPath: .status.backup.version - description: Backup Version - name: Version - type: string - - JSONPath: .status.backup.createdAt - description: Backup Creation Timestamp - name: Created + - JSONPath: .spec.schedule + description: Schedule + name: Schedule type: string - - JSONPath: .status.state - description: The actual state of the ArangoBackup - name: State + - JSONPath: .status.scheduled + description: Scheduled + name: Scheduled type: string - JSONPath: .status.message priority: 1 - description: Message of the ArangoBackup object + description: Message of the ArangoBackupPolicy object name: Message type: string names: - kind: ArangoBackup - listKind: ArangoBackupList - plural: arangobackups + kind: ArangoBackupPolicy + listKind: ArangoBackupPolicyList + plural: arangobackuppolicies shortNames: - - arangobackup - singular: arangobackup + - arangobackuppolicy + - arangobp + singular: arangobackuppolicy scope: Namespaced version: v1alpha subresources: status: {} + --- # Source: kube-arangodb-crd/templates/backup.yaml apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: - name: arangobackuppolicies.backup.arangodb.com + name: arangobackups.backup.arangodb.com labels: app.kubernetes.io/name: kube-arangodb-crd helm.sh/chart: kube-arangodb-crd-0.3.16 @@ -64,32 +54,51 @@ metadata: spec: group: backup.arangodb.com additionalPrinterColumns: - - JSONPath: .spec.schedule - description: Schedule - name: Schedule + - JSONPath: .spec.policyName + description: Policy name + name: Policy type: string - - JSONPath: .status.scheduled - description: Scheduled - name: Scheduled + - JSONPath: .spec.deployment.name + description: Deployment name + name: Deployment + type: string + - JSONPath: .status.backup.version + description: Backup Version + name: Version + type: string + - JSONPath: .status.backup.createdAt + description: Backup Creation Timestamp + name: Created + type: string + - JSONPath: .status.backup.sizeInBytes + description: Backup Size in Bytes + name: Size + type: integer + format: byte + - JSONPath: .status.backup.numberOfDBServers + description: Backup Number of the DB Servers + name: DBServers + type: integer + - JSONPath: .status.state + description: The actual state of the ArangoBackup + name: State type: string - JSONPath: .status.message priority: 1 - description: Message of the ArangoBackupPolicy object + description: Message of the ArangoBackup object name: Message type: string names: - kind: ArangoBackupPolicy - listKind: ArangoBackupPolicyList - plural: arangobackuppolicies + kind: ArangoBackup + listKind: ArangoBackupList + plural: arangobackups shortNames: - - arangobackuppolicy - - arangobp - singular: arangobackuppolicy + - arangobackup + singular: arangobackup scope: Namespaced version: v1alpha subresources: status: {} - --- # Source: kube-arangodb-crd/templates/deployment-replications.yaml apiVersion: apiextensions.k8s.io/v1beta1 diff --git a/pkg/apis/backup/v1alpha/backup_state.go b/pkg/apis/backup/v1alpha/backup_state.go index 0b805059b..2d5c609cd 100644 --- a/pkg/apis/backup/v1alpha/backup_state.go +++ b/pkg/apis/backup/v1alpha/backup_state.go @@ -41,22 +41,24 @@ const ( ArangoBackupStateReady state.State = "Ready" ArangoBackupStateDeleted state.State = "Deleted" ArangoBackupStateFailed state.State = "Failed" + ArangoBackupStateUnavailable state.State = "Unavailable" ) var ArangoBackupStateMap = state.Map{ - ArangoBackupStateNone: {ArangoBackupStatePending, ArangoBackupStateFailed}, + ArangoBackupStateNone: {ArangoBackupStatePending}, ArangoBackupStatePending: {ArangoBackupStateScheduled, ArangoBackupStateFailed}, ArangoBackupStateScheduled: {ArangoBackupStateDownload, ArangoBackupStateCreate, ArangoBackupStateFailed}, - ArangoBackupStateDownload: {ArangoBackupStateDownloading, ArangoBackupStateFailed}, + ArangoBackupStateDownload: {ArangoBackupStateDownloading, ArangoBackupStateFailed, ArangoBackupStateDownloadError}, ArangoBackupStateDownloading: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateDownloadError}, ArangoBackupStateDownloadError: {ArangoBackupStatePending, ArangoBackupStateFailed}, ArangoBackupStateCreate: {ArangoBackupStateReady, ArangoBackupStateFailed}, - ArangoBackupStateUpload: {ArangoBackupStateUploading, ArangoBackupStateFailed}, + ArangoBackupStateUpload: {ArangoBackupStateUploading, ArangoBackupStateFailed, ArangoBackupStateDeleted, ArangoBackupStateUploadError}, ArangoBackupStateUploading: {ArangoBackupStateReady, ArangoBackupStateFailed, ArangoBackupStateUploadError}, ArangoBackupStateUploadError: {ArangoBackupStateFailed, ArangoBackupStateReady}, - ArangoBackupStateReady: {ArangoBackupStateDeleted, ArangoBackupStateFailed, ArangoBackupStateUpload}, - ArangoBackupStateDeleted: {ArangoBackupStateFailed}, + ArangoBackupStateReady: {ArangoBackupStateDeleted, ArangoBackupStateFailed, ArangoBackupStateUpload, ArangoBackupStateUnavailable}, + ArangoBackupStateDeleted: {ArangoBackupStateFailed, ArangoBackupStateReady}, ArangoBackupStateFailed: {ArangoBackupStatePending}, + ArangoBackupStateUnavailable: {ArangoBackupStateReady, ArangoBackupStateDeleted, ArangoBackupStateFailed}, } type ArangoBackupState struct { @@ -72,7 +74,35 @@ type ArangoBackupState struct { Progress *ArangoBackupProgress `json:"progress,omitempty"` } +func (a *ArangoBackupState) Equal(b *ArangoBackupState) bool { + if a == b { + return true + } + + if a == nil && b != nil || a != nil && b == nil { + return false + } + + return a.State == b.State && + a.Time.Equal(&b.Time) && + a.Message == b.Message && + a.Progress.Equal(b.Progress) +} + type ArangoBackupProgress struct { JobID string `json:"jobID"` Progress string `json:"progress"` } + +func (a *ArangoBackupProgress) Equal(b *ArangoBackupProgress) bool { + if a == b { + return true + } + + if a == nil && b != nil || a != nil && b == nil { + return false + } + + return a.JobID == b.JobID && + a.Progress == b.Progress +} diff --git a/pkg/apis/backup/v1alpha/backup_status.go b/pkg/apis/backup/v1alpha/backup_status.go index b5cfdbab4..bc54d2f1a 100644 --- a/pkg/apis/backup/v1alpha/backup_status.go +++ b/pkg/apis/backup/v1alpha/backup_status.go @@ -34,12 +34,64 @@ type ArangoBackupStatus struct { Available bool `json:"available"` } +func (a *ArangoBackupStatus) Equal(b *ArangoBackupStatus) bool { + if a == b { + return true + } + + if a == nil && b != nil || a != nil && b == nil { + return false + } + + return a.ArangoBackupState.Equal(&b.ArangoBackupState) && + a.Backup.Equal(b.Backup) && + a.Available == b.Available +} + type ArangoBackupDetails struct { ID string `json:"id"` Version string `json:"version"` PotentiallyInconsistent *bool `json:"potentiallyInconsistent,omitempty"` + SizeInBytes uint64 `json:"sizeInBytes,omitempty"` + NumberOfDBServers uint `json:"numberOfDBServers,omitempty"` Uploaded *bool `json:"uploaded,omitempty"` Downloaded *bool `json:"downloaded,omitempty"` Imported *bool `json:"imported,omitempty"` CreationTimestamp meta.Time `json:"createdAt"` } + +func (a *ArangoBackupDetails) Equal(b *ArangoBackupDetails) bool { + if a == b { + return true + } + + if a == nil && b != nil || a != nil && b == nil { + return false + } + + return a.ID == b.ID && + a.Version == b.Version && + a.SizeInBytes == b.SizeInBytes && + a.NumberOfDBServers == b.NumberOfDBServers && + a.CreationTimestamp.Equal(&b.CreationTimestamp) && + compareBoolPointer(a.PotentiallyInconsistent, b.PotentiallyInconsistent) && + compareBoolPointer(a.Uploaded, b.Uploaded) && + compareBoolPointer(a.Downloaded, b.Downloaded) && + compareBoolPointer(a.Imported, b.Imported) +} + +func compareBoolPointer(a, b * bool) bool { + if a == nil && b != nil || a != nil && b == nil { + return false + } + + if a == b { + return true + } + + if *a == *b { + return true + } + + return false +} \ No newline at end of file diff --git a/pkg/backup/handlers/arango/backup/arango_client.go b/pkg/backup/handlers/arango/backup/arango_client.go index d7cccf635..09464b780 100644 --- a/pkg/backup/handlers/arango/backup/arango_client.go +++ b/pkg/backup/handlers/arango/backup/arango_client.go @@ -23,11 +23,8 @@ package backup import ( - "fmt" - "net/http" - "strings" - "github.com/arangodb/kube-arangodb/pkg/backup/utils" + "net/http" "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" @@ -75,59 +72,4 @@ type ArangoBackupClient interface { Delete(driver.BackupID) error List() (map[driver.BackupID]driver.BackupMeta, error) -} - -// NewTemporaryError created new temporary error -func NewTemporaryError(format string, a ...interface{}) error { - return TemporaryError{ - Message: fmt.Sprintf(format, a...), - } -} - -// TemporaryError defines error which will not update ArangoBackup object status -type TemporaryError struct { - Message string -} - -func (t TemporaryError) Error() string { - return t.Message -} - -// IsTemporaryError determined if error is type of TemporaryError -func IsTemporaryError(err error) bool { - _, ok := err.(TemporaryError) - return ok -} - -func checkTemporaryError(err error) bool { - if ok := IsTemporaryError(err); ok { - return true - } - - if v, ok := err.(utils.Temporary); ok { - if v.Temporary() { - return true - } - } - - if v, ok := err.(driver.ArangoError); ok { - if temporaryErrorNum.Has(v.ErrorNum) || temporaryCodes.Has(v.Code) { - return true - } - } - - if v, ok := err.(utils.Causer); ok { - return checkTemporaryError(v.Cause()) - } - - // Check error string - if strings.Contains(err.Error(), "context deadline exceeded") { - return true - } - - if strings.Contains(err.Error(), "connection refused") { - return true - } - - return false -} +} \ No newline at end of file diff --git a/pkg/backup/handlers/arango/backup/arango_client_mock_test.go b/pkg/backup/handlers/arango/backup/arango_client_mock_test.go index 81caee74f..5f7526017 100644 --- a/pkg/backup/handlers/arango/backup/arango_client_mock_test.go +++ b/pkg/backup/handlers/arango/backup/arango_client_mock_test.go @@ -24,7 +24,9 @@ package backup import ( "fmt" + "math/rand" "sync" + "time" "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" @@ -42,14 +44,17 @@ func newMockArangoClientBackupErrorFactory(err error) ArangoClientFactory { } } -func newMockArangoClientBackupFactory(mock *mockArangoClientBackup) ArangoClientFactory { +func newMockArangoClientBackupFactory(mock *mockArangoClientBackupState) ArangoClientFactory { return func(deployment *database.ArangoDeployment, backup *backupApi.ArangoBackup) (ArangoBackupClient, error) { - return mock, nil + return &mockArangoClientBackup{ + backup: backup, + state: mock, + }, nil } } -func newMockArangoClientBackup(errors mockErrorsArangoClientBackup) *mockArangoClientBackup { - return &mockArangoClientBackup{ +func newMockArangoClientBackup(errors mockErrorsArangoClientBackup) *mockArangoClientBackupState { + return &mockArangoClientBackupState{ backups: map[driver.BackupID]driver.BackupMeta{}, progresses: map[driver.BackupTransferJobID]ArangoBackupProgress{}, errors: errors, @@ -57,13 +62,10 @@ func newMockArangoClientBackup(errors mockErrorsArangoClientBackup) *mockArangoC } type mockErrorsArangoClientBackup struct { - createError, listError, getError, uploadError, downloadError, progressError, existsError, deleteError, abortError string - isTemporaryError bool - - createForced bool + createError, listError, getError, uploadError, downloadError, progressError, existsError, deleteError, abortError error } -type mockArangoClientBackup struct { +type mockArangoClientBackupState struct { lock sync.Mutex backups map[driver.BackupID]driver.BackupMeta @@ -72,121 +74,115 @@ type mockArangoClientBackup struct { errors mockErrorsArangoClientBackup } +type mockArangoClientBackup struct { + backup *backupApi.ArangoBackup + state *mockArangoClientBackupState +} + func (m *mockArangoClientBackup) List() (map[driver.BackupID]driver.BackupMeta, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.listError); err != nil { - return nil, err + if m.state.errors.listError != nil { + return nil, m.state.errors.listError } - return m.backups, nil + return m.state.backups, nil } func (m *mockArangoClientBackup) Abort(d driver.BackupTransferJobID) error { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.abortError); err != nil { - return err + if m.state.errors.abortError != nil { + return m.state.errors.abortError } - if _, ok := m.progresses[d]; ok { - delete(m.progresses, d) + if _, ok := m.state.progresses[d]; ok { + delete(m.state.progresses, d) } return nil } func (m *mockArangoClientBackup) Exists(id driver.BackupID) (bool, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.existsError); err != nil { - return false, err + if m.state.errors.existsError != nil { + return false, m.state.errors.existsError } - _, ok := m.backups[id] + _, ok := m.state.backups[id] return ok, nil } func (m *mockArangoClientBackup) Delete(id driver.BackupID) error { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.deleteError); err != nil { - return err + if m.state.errors.deleteError != nil { + return m.state.errors.deleteError } - if _, ok := m.backups[id]; ok { - delete(m.backups, id) + if _, ok := m.state.backups[id]; ok { + delete(m.state.backups, id) } return nil } -func (m *mockArangoClientBackup) newError(msg string) error { - if msg == "" { - return nil - } - - if m.errors.isTemporaryError { - return NewTemporaryError(msg) - } - return fmt.Errorf(msg) -} - func (m *mockArangoClientBackup) Download(driver.BackupID) (driver.BackupTransferJobID, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.downloadError); err != nil { - return "", err + if m.state.errors.downloadError != nil { + return "", m.state.errors.downloadError } id := driver.BackupTransferJobID(uuid.NewUUID()) - m.progresses[id] = ArangoBackupProgress{} + m.state.progresses[id] = ArangoBackupProgress{} return id, nil } func (m *mockArangoClientBackup) Progress(id driver.BackupTransferJobID) (ArangoBackupProgress, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.progressError); err != nil { - return ArangoBackupProgress{}, err + if m.state.errors.progressError != nil { + return ArangoBackupProgress{}, m.state.errors.progressError } - return m.progresses[id], nil + return m.state.progresses[id], nil } func (m *mockArangoClientBackup) Upload(driver.BackupID) (driver.BackupTransferJobID, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.uploadError); err != nil { - return "", err + if m.state.errors.uploadError != nil { + return "", m.state.errors.uploadError } id := driver.BackupTransferJobID(uuid.NewUUID()) - m.progresses[id] = ArangoBackupProgress{} + m.state.progresses[id] = ArangoBackupProgress{} return id, nil } func (m *mockArangoClientBackup) Get(id driver.BackupID) (driver.BackupMeta, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.getError); err != nil { - return driver.BackupMeta{}, err + if m.state.errors.getError != nil { + return driver.BackupMeta{}, m.state.errors.getError } - if meta, ok := m.backups[id]; ok { + if meta, ok := m.state.backups[id]; ok { return meta, nil } @@ -194,32 +190,48 @@ func (m *mockArangoClientBackup) Get(id driver.BackupID) (driver.BackupMeta, err } func (m *mockArangoClientBackup) Create() (ArangoBackupCreateResponse, error) { - m.lock.Lock() - defer m.lock.Unlock() + m.state.lock.Lock() + defer m.state.lock.Unlock() - if err := m.newError(m.errors.createError); err != nil { - return ArangoBackupCreateResponse{}, err + if m.state.errors.createError != nil { + return ArangoBackupCreateResponse{}, m.state.errors.createError } id := driver.BackupID(uuid.NewUUID()) + inconsistent := false + + if m.backup != nil { + if m.backup.Spec.Options != nil { + if m.backup.Spec.Options.AllowInconsistent != nil { + inconsistent = *m.backup.Spec.Options.AllowInconsistent + } + } + } + meta := driver.BackupMeta{ - ID: id, - Version: mockVersion, + ID: id, + Version: mockVersion, + NumberOfDBServers: uint(rand.Uint32()), + DateTime: time.Now(), + SizeInBytes: rand.Uint64(), + PotentiallyInconsistent: inconsistent, + NumberOfFiles: uint(rand.Uint32()), + Available: true, } - m.backups[id] = meta + m.state.backups[id] = meta return ArangoBackupCreateResponse{ BackupMeta: meta, - PotentiallyInconsistent: m.errors.createForced, + PotentiallyInconsistent: meta.PotentiallyInconsistent, }, nil } func (m *mockArangoClientBackup) getIDs() []string { - ret := make([]string, 0, len(m.backups)) + ret := make([]string, 0, len(m.state.backups)) - for key := range m.backups { + for key := range m.state.backups { ret = append(ret, string(key)) } @@ -227,9 +239,9 @@ func (m *mockArangoClientBackup) getIDs() []string { } func (m *mockArangoClientBackup) getProgressIDs() []string { - ret := make([]string, 0, len(m.progresses)) + ret := make([]string, 0, len(m.state.progresses)) - for key := range m.progresses { + for key := range m.state.progresses { ret = append(ret, string(key)) } diff --git a/pkg/backup/handlers/arango/backup/arango_client_test.go b/pkg/backup/handlers/arango/backup/arango_client_test.go index 3dd59a2ee..0b12d80e9 100644 --- a/pkg/backup/handlers/arango/backup/arango_client_test.go +++ b/pkg/backup/handlers/arango/backup/arango_client_test.go @@ -100,7 +100,7 @@ func Test_Errors_Temporary(t *testing.T) { // Act for testName, c := range checks { t.Run(testName, func(t *testing.T) { - res := checkTemporaryError(c.err) + res := isTemporaryError(c.err) if c.temporary { require.True(t, res) } else { diff --git a/pkg/backup/handlers/arango/backup/backup_suite_test.go b/pkg/backup/handlers/arango/backup/backup_suite_test.go index f2434549f..15f92332d 100644 --- a/pkg/backup/handlers/arango/backup/backup_suite_test.go +++ b/pkg/backup/handlers/arango/backup/backup_suite_test.go @@ -24,6 +24,7 @@ package backup import ( "fmt" + "github.com/arangodb/go-driver" "testing" "github.com/arangodb/kube-arangodb/pkg/backup/operator/event" @@ -62,7 +63,10 @@ func newErrorsFakeHandler(errors mockErrorsArangoClientBackup) (*handler, *mockA mock := newMockArangoClientBackup(errors) handler.arangoClientFactory = newMockArangoClientBackupFactory(mock) - return handler, mock + return handler, &mockArangoClientBackup{ + backup: nil, + state: mock, + } } func newObjectSet(state state.State) (*backupApi.ArangoBackup, *database.ArangoDeployment) { @@ -77,7 +81,7 @@ func newObjectSet(state state.State) (*backupApi.ArangoBackup, *database.ArangoD func compareTemporaryState(t *testing.T, err error, errorMsg string, handler *handler, obj *backupApi.ArangoBackup) { require.Error(t, err) - require.True(t, IsTemporaryError(err)) + require.True(t, isTemporaryError(err)) require.EqualError(t, err, errorMsg) newObj := refreshArangoBackup(t, handler, obj) @@ -170,6 +174,21 @@ func createArangoDeployment(t *testing.T, h *handler, deployments ...*database.A } } +func compareBackupMeta(t *testing.T, backupMeta driver.BackupMeta, backup *backupApi.ArangoBackup) { + require.NotNil(t, backup.Status.Backup) + require.Equal(t, string(backupMeta.ID), backup.Status.Backup.ID) + require.Equal(t, backupMeta.PotentiallyInconsistent, *backup.Status.Backup.PotentiallyInconsistent) + require.Equal(t, backupMeta.SizeInBytes, backup.Status.Backup.SizeInBytes) + require.Equal(t, backupMeta.DateTime.UTC().Unix(), backup.Status.Backup.CreationTimestamp.Time.UTC().Unix()) + require.Equal(t, backupMeta.NumberOfDBServers, backup.Status.Backup.NumberOfDBServers) + require.Equal(t, backupMeta.Version, backup.Status.Backup.Version) +} + +func checkBackup(t *testing.T, backup *backupApi.ArangoBackup, state state.State, available bool) { + require.Equal(t, state, backup.Status.State) + require.Equal(t, available, backup.Status.Available) +} + func wrapperUndefinedDeployment(t *testing.T, state state.State) { t.Run("Empty Name", func(t *testing.T) { // Arrange @@ -186,7 +205,7 @@ func wrapperUndefinedDeployment(t *testing.T, state state.State) { newObj := refreshArangoBackup(t, handler, obj) require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) - require.Equal(t, newObj.Status.Message, createFailMessage(state, "deployment name can not be empty")) + require.Equal(t, newObj.Status.Message, createStateMessage(state, backupApi.ArangoBackupStateFailed, "deployment name can not be empty")) }) t.Run("Missing Deployment", func(t *testing.T) { @@ -203,7 +222,7 @@ func wrapperUndefinedDeployment(t *testing.T, state state.State) { newObj := refreshArangoBackup(t, handler, obj) require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) - require.Equal(t, newObj.Status.Message, createFailMessage(state, fmt.Sprintf("%s \"%s\" not found", database.ArangoDeploymentCRDName, obj.Name))) + require.Equal(t, newObj.Status.Message, createStateMessage(state, backupApi.ArangoBackupStateFailed, fmt.Sprintf("%s \"%s\" not found", database.ArangoDeploymentCRDName, obj.Name))) }) } @@ -224,7 +243,7 @@ func wrapperConnectionIssues(t *testing.T, state state.State) { // Assert require.Error(t, err) - require.True(t, IsTemporaryError(err)) + require.True(t, isTemporaryError(err)) newObj := refreshArangoBackup(t, handler, obj) require.Equal(t, newObj.Status.State, state) @@ -248,7 +267,7 @@ func wrapperProgressMissing(t *testing.T, state state.State) { newObj := refreshArangoBackup(t, handler, obj) require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) - require.Equal(t, newObj.Status.Message, createFailMessage(state, fmt.Sprintf("backup details are missing"))) + require.Equal(t, newObj.Status.Message, createStateMessage(state, backupApi.ArangoBackupStateFailed, fmt.Sprintf("missing field .status.backup"))) }) } diff --git a/pkg/backup/handlers/arango/backup/errors.go b/pkg/backup/handlers/arango/backup/errors.go new file mode 100644 index 000000000..9ecf0e995 --- /dev/null +++ b/pkg/backup/handlers/arango/backup/errors.go @@ -0,0 +1,119 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package backup + +import ( + "fmt" + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/backup/utils" + "strings" +) + +func newTemporaryError(err error) error { + return temporaryError{ + Causer:err, + } +} + +func newTemporaryErrorf(format string, a ... interface{}) error { + return newTemporaryError(fmt.Errorf(format, a...)) +} + +type temporaryError struct { + Causer error +} + +func (t temporaryError) Cause() error { + return t.Causer +} + +func (t temporaryError) Error() string { + return t.Causer.Error() +} + +func newFatalError(err error) error { + return fatalError{ + Causer:err, + } +} + +func newFatalErrorf(format string, a ... interface{}) error { + return newFatalError(fmt.Errorf(format, a...)) +} + +type fatalError struct { + Causer error +} + +func (f fatalError) Cause() error { + return f.Causer +} + +func (f fatalError) Error() string { + return f.Causer.Error() +} + +func isTemporaryError(err error) bool { + if _, ok := err.(temporaryError); ok { + return true + } + + if _, ok := err.(fatalError); ok { + return false + } + + if v, ok := err.(utils.Temporary); ok { + if v.Temporary() { + return true + } + } + + if v, ok := err.(driver.ArangoError); ok { + if temporaryErrorNum.Has(v.ErrorNum) || temporaryCodes.Has(v.Code) { + return true + } + } + + if v, ok := err.(utils.Causer); ok { + return isTemporaryError(v.Cause()) + } + + // Check error string + if strings.Contains(err.Error(), "context deadline exceeded") { + return true + } + + if strings.Contains(err.Error(), "connection refused") { + return true + } + + return false +} + +func switchError(err error) error { + if isTemporaryError(err) { + return newTemporaryError(err) + } + + return newFatalError(err) +} \ No newline at end of file diff --git a/pkg/backup/handlers/arango/backup/finalizer.go b/pkg/backup/handlers/arango/backup/finalizer.go index 7e3b91db9..dae109afe 100644 --- a/pkg/backup/handlers/arango/backup/finalizer.go +++ b/pkg/backup/handlers/arango/backup/finalizer.go @@ -28,6 +28,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/backup/utils" "github.com/rs/zerolog/log" "k8s.io/apimachinery/pkg/api/errors" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) func (h *handler) finalize(backup *backupApi.ArangoBackup) error { @@ -68,6 +69,10 @@ func (h *handler) finalize(backup *backupApi.ArangoBackup) error { } func (h *handler) finalizeBackup(backup *backupApi.ArangoBackup) error { + lock := h.getDeploymentMutex(backup.Namespace, backup.Spec.Deployment.Name) + lock.Lock() + defer lock.Unlock() + if backup.Status.Backup == nil { // No details passed, object can be removed return nil @@ -80,9 +85,35 @@ func (h *handler) finalizeBackup(backup *backupApi.ArangoBackup) error { return nil } + if c, ok := err.(utils.Causer); ok { + if errors.IsNotFound(c.Cause()) { + return nil + } + } + + return err + } + + backups, err := h.client.BackupV1alpha().ArangoBackups(backup.Namespace).List(meta.ListOptions{}) + if err != nil { return err } + for _, existingBackup := range backups.Items { + if existingBackup.Name == backup.Name { + continue + } + + if existingBackup.Status.Backup == nil { + continue + } + + // This backup is still in use + if existingBackup.Status.Backup.ID == backup.Status.Backup.ID { + return nil + } + } + client, err := h.arangoClientFactory(deployment, backup) if err != nil { return err diff --git a/pkg/backup/handlers/arango/backup/handler.go b/pkg/backup/handlers/arango/backup/handler.go index c78011ff2..49238ef4c 100644 --- a/pkg/backup/handlers/arango/backup/handler.go +++ b/pkg/backup/handlers/arango/backup/handler.go @@ -25,7 +25,6 @@ package backup import ( "fmt" "github.com/arangodb/kube-arangodb/pkg/util" - "reflect" "sync" "time" @@ -178,19 +177,12 @@ func (h *handler) refreshDeploymentBackup(deployment *database.ArangoDeployment, return err } - backup.Status = backupApi.ArangoBackupStatus{ - Backup: &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - Imported: util.NewBool(true), - }, - Available: true, - ArangoBackupState: backupApi.ArangoBackupState{ - Time: meta.Now(), - State: backupApi.ArangoBackupStateReady, - }, - } + status := updateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady, ""), + updateStatusBackup(backupMeta), + updateStatusBackupImported(util.NewBool(true))) + + backup.Status = *status err = h.updateBackupStatus(backup) if err != nil { @@ -288,6 +280,15 @@ func (h *handler) Handle(item operation.Item) error { return err } } + + b, err = h.client.BackupV1alpha().ArangoBackups(item.Namespace).Get(item.Name, meta.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + return nil + } + + return err + } } status, err := h.processArangoBackup(b.DeepCopy()) @@ -296,13 +297,22 @@ func (h *handler) Handle(item operation.Item) error { item.Kind, item.Namespace, item.Name) - return err + + cError := switchError(err) + + if _, ok := cError.(temporaryError); ok { + return cError + } + + status, _ = setFailedState(b, cError) } - status.Time = b.Status.Time + if status == nil { + return nil + } // Nothing to update, objects are equal - if reflect.DeepEqual(b.Status, status) { + if b.Status.Equal(status) { return nil } @@ -317,7 +327,6 @@ func (h *handler) Handle(item operation.Item) error { // Log message about state change if b.Status.State != status.State { - status.Time = meta.Now() if status.State == backupApi.ArangoBackupStateFailed { h.eventRecorder.Warning(b, StateChange, "Transiting from %s to %s with error: %s", b.Status.State, @@ -330,7 +339,7 @@ func (h *handler) Handle(item operation.Item) error { } } - b.Status = status + b.Status = *status log.Debug().Msgf("Updating %s %s/%s", item.Kind, @@ -345,16 +354,16 @@ func (h *handler) Handle(item operation.Item) error { return nil } -func (h *handler) processArangoBackup(backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func (h *handler) processArangoBackup(backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { if err := backup.Validate(); err != nil { - return createFailedState(err, backup.Status), nil + return setFailedState(backup, err) } if f, ok := stateHolders[backup.Status.State]; ok { return f(h, backup) } - return backupApi.ArangoBackupStatus{}, fmt.Errorf("state %s is not supported", backup.Status.State) + return nil, fmt.Errorf("state %s is not supported", backup.Status.State) } func (h *handler) CanBeHandled(item operation.Item) bool { @@ -365,8 +374,19 @@ func (h *handler) CanBeHandled(item operation.Item) bool { func (h *handler) getArangoDeploymentObject(backup *backupApi.ArangoBackup) (*database.ArangoDeployment, error) { if backup.Spec.Deployment.Name == "" { - return nil, fmt.Errorf("deployment ref is not specified for backup %s/%s", backup.Namespace, backup.Name) + return nil, newFatalErrorf("deployment ref is not specified for backup %s/%s", backup.Namespace, backup.Name) + } + + obj, err := h.client.DatabaseV1alpha().ArangoDeployments(backup.Namespace).Get(backup.Spec.Deployment.Name, meta.GetOptions{}) + if err == nil { + return obj, nil + } + + // Check if object is not found + if errors.IsNotFound(err) { + return nil, newFatalError(err) } - return h.client.DatabaseV1alpha().ArangoDeployments(backup.Namespace).Get(backup.Spec.Deployment.Name, meta.GetOptions{}) + // Otherwise it is connection issue - mark as temporary + return nil, newTemporaryError(err) } diff --git a/pkg/backup/handlers/arango/backup/state.go b/pkg/backup/handlers/arango/backup/state.go index 2a1189677..c13c3a516 100644 --- a/pkg/backup/handlers/arango/backup/state.go +++ b/pkg/backup/handlers/arango/backup/state.go @@ -27,7 +27,7 @@ import ( "github.com/arangodb/kube-arangodb/pkg/backup/state" ) -type stateHolder func(handler *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) +type stateHolder func(handler *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) var ( stateHolders = map[state.State]stateHolder{ @@ -44,5 +44,6 @@ var ( backupApi.ArangoBackupStateReady: stateReadyHandler, backupApi.ArangoBackupStateDeleted: stateDeletedHandler, backupApi.ArangoBackupStateFailed: stateFailedHandler, + backupApi.ArangoBackupStateUnavailable: stateUnavailableHandler, } ) diff --git a/pkg/backup/handlers/arango/backup/state_create.go b/pkg/backup/handlers/arango/backup/state_create.go index fe8be2f1e..851865635 100644 --- a/pkg/backup/handlers/arango/backup/state_create.go +++ b/pkg/backup/handlers/arango/backup/state_create.go @@ -23,43 +23,42 @@ package backup import ( + "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateCreateHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { deployment, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } client, err := h.arangoClientFactory(deployment, backup) if err != nil { - return backupApi.ArangoBackupStatus{}, NewTemporaryError("unable to create client: %s", err.Error()) + return nil, newTemporaryError(err) } - var details *backupApi.ArangoBackupDetails - - // Try to recover old backup. If old backup is missing go into deleted state - response, err := client.Create() if err != nil { - return switchTemporaryError(err, backup.Status) + return nil, err } - details = &backupApi.ArangoBackupDetails{ - ID: string(response.ID), - Version: response.Version, - CreationTimestamp: meta.Now(), - } + backupMeta, err := client.Get(response.ID) + if err != nil { + if driver.IsNotFound(err) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateFailed, + "backup is not present after creation"), + cleanStatusJob(), + ) + } - if response.PotentiallyInconsistent { - details.PotentiallyInconsistent = &response.PotentiallyInconsistent + return nil, newFatalError(err) } - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: details, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady,""), + updateStatusAvailable(true), + updateStatusBackup(backupMeta), + ) } diff --git a/pkg/backup/handlers/arango/backup/state_create_test.go b/pkg/backup/handlers/arango/backup/state_create_test.go index 8c856d71a..303a09495 100644 --- a/pkg/backup/handlers/arango/backup/state_create_test.go +++ b/pkg/backup/handlers/arango/backup/state_create_test.go @@ -23,6 +23,8 @@ package backup import ( + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/util" "testing" "github.com/arangodb/kube-arangodb/pkg/backup/operator/operation" @@ -50,25 +52,25 @@ func Test_State_Create_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateReady) - - require.NotNil(t, newObj.Status.Backup) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) backups := mock.getIDs() require.Len(t, backups, 1) - require.Equal(t, newObj.Status.Backup.ID, backups[0]) - require.Equal(t, newObj.Status.Backup.Version, mockVersion) + backupMeta, err := mock.Get(driver.BackupID(backups[0])) + require.NoError(t, err) - require.Nil(t, newObj.Status.Backup.PotentiallyInconsistent) + compareBackupMeta(t, backupMeta, newObj) } func Test_State_Create_SuccessForced(t *testing.T) { // Arrange handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) - mock.errors.createForced = true obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreate) + obj.Spec.Options = &backupApi.ArangoBackupSpecOptions{ + AllowInconsistent: util.NewBool(true), + } // Act createArangoDeployment(t, handler, deployment) @@ -78,19 +80,17 @@ func Test_State_Create_SuccessForced(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateReady) - - require.NotNil(t, newObj.Status.Backup) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) backups := mock.getIDs() require.Len(t, backups, 1) - require.Equal(t, newObj.Status.Backup.ID, backups[0]) - require.Equal(t, newObj.Status.Backup.Version, mockVersion) + backupMeta, err := mock.Get(driver.BackupID(backups[0])) + require.NoError(t, err) + compareBackupMeta(t, backupMeta, newObj) require.NotNil(t, newObj.Status.Backup.PotentiallyInconsistent) - value := *newObj.Status.Backup.PotentiallyInconsistent - require.True(t, value) + require.True(t, *newObj.Status.Backup.PotentiallyInconsistent) } func Test_State_Create_Upload(t *testing.T) { @@ -110,24 +110,22 @@ func Test_State_Create_Upload(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateReady) - - require.NotNil(t, newObj.Status.Backup) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) backups := mock.getIDs() require.Len(t, backups, 1) - require.Equal(t, newObj.Status.Backup.ID, backups[0]) - require.Equal(t, newObj.Status.Backup.Version, mockVersion) + backupMeta, err := mock.Get(driver.BackupID(backups[0])) + require.NoError(t, err) - require.True(t, newObj.Status.Available) + compareBackupMeta(t, backupMeta, newObj) } func Test_State_Create_CreateFailed(t *testing.T) { // Arrange - errorMsg := "create error" + error := newFatalErrorf("error") handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - createError: errorMsg, + createError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreate) @@ -141,7 +139,7 @@ func Test_State_Create_CreateFailed(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) - require.Equal(t, newObj.Status.Message, createFailMessage(backupApi.ArangoBackupStateCreate, errorMsg)) + require.Equal(t, newObj.Status.Message, createStateMessage(backupApi.ArangoBackupStateCreate, backupApi.ArangoBackupStateFailed, error.Error())) require.Nil(t, newObj.Status.Backup) @@ -150,10 +148,9 @@ func Test_State_Create_CreateFailed(t *testing.T) { func Test_State_Create_TemporaryCreateFailed(t *testing.T) { // Arrange - errorMsg := "create error" + error := newTemporaryErrorf("error") handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - isTemporaryError: true, - createError: errorMsg, + createError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateCreate) @@ -163,7 +160,10 @@ func Test_State_Create_TemporaryCreateFailed(t *testing.T) { createArangoBackup(t, handler, obj) err := handler.Handle(newItemFromBackup(operation.Update, obj)) + require.EqualError(t, err, error.Error()) // Assert - compareTemporaryState(t, err, errorMsg, handler, obj) + newObj := refreshArangoBackup(t, handler, obj) + + require.Equal(t, obj.Status, newObj.Status) } diff --git a/pkg/backup/handlers/arango/backup/state_deleted.go b/pkg/backup/handlers/arango/backup/state_deleted.go index 78af7ccf6..f7c544697 100644 --- a/pkg/backup/handlers/arango/backup/state_deleted.go +++ b/pkg/backup/handlers/arango/backup/state_deleted.go @@ -23,9 +23,32 @@ package backup import ( + "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateDeletedHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { - return backup.Status, nil +func stateDeletedHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + deployment, err := h.getArangoDeploymentObject(backup) + if err != nil { + return nil, err + } + + client, err := h.arangoClientFactory(deployment, backup) + if err != nil { + return nil, newTemporaryError(err) + } + + if backup.Status.Backup != nil { + backupMeta, err := client.Get(driver.BackupID(backup.Status.Backup.ID)) + if err == nil { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady, ""), + updateStatusAvailable(true), + updateStatusBackup(backupMeta), + ) + } + } + + return wrapUpdateStatus(backup, + updateStatusAvailable(false)) } diff --git a/pkg/backup/handlers/arango/backup/state_deleted_test.go b/pkg/backup/handlers/arango/backup/state_deleted_test.go index aeb181b84..afbd10eeb 100644 --- a/pkg/backup/handlers/arango/backup/state_deleted_test.go +++ b/pkg/backup/handlers/arango/backup/state_deleted_test.go @@ -46,4 +46,31 @@ func Test_State_Deleted(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) require.Equal(t, newObj.Status, obj.Status) + checkBackup(t, newObj, backupApi.ArangoBackupStateDeleted, false) +} + +func Test_State_Deleted_Recover(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateDeleted) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) } diff --git a/pkg/backup/handlers/arango/backup/state_download.go b/pkg/backup/handlers/arango/backup/state_download.go index 8a1eeefc4..9a0f67a3b 100644 --- a/pkg/backup/handlers/arango/backup/state_download.go +++ b/pkg/backup/handlers/arango/backup/state_download.go @@ -23,43 +23,53 @@ package backup import ( - "fmt" - "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateDownloadHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateDownloadHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { deployment, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } client, err := h.arangoClientFactory(deployment, backup) if err != nil { - return backupApi.ArangoBackupStatus{}, NewTemporaryError("unable to create client: %s", err.Error()) + return nil, newTemporaryError(err) } if backup.Spec.Download == nil { - return createFailedState(fmt.Errorf("missing field .spec.download"), backup.Status), nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloadError, + "missing field .spec.download"), + cleanStatusJob(), + updateStatusAvailable(false), + ) } if backup.Spec.Download.ID == "" { - return createFailedState(fmt.Errorf("missing field .spec.download.id"), backup.Status), nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloadError, + "missing field .spec.download.id"), + cleanStatusJob(), + updateStatusAvailable(false), + ) } jobID, err := client.Download(driver.BackupID(backup.Spec.Download.ID)) if err != nil { - return switchTemporaryError(err, backup.Status) + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloadError, + "missing field .spec.download.id"), + cleanStatusJob(), + updateStatusAvailable(false), + ) } - return backupApi.ArangoBackupStatus{ - Available: false, - ArangoBackupState: newState(backupApi.ArangoBackupStateDownloading, "", - &backupApi.ArangoBackupProgress{ - JobID: string(jobID), - Progress: "0%", - }), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloading, ""), + updateStatusJob(string(jobID), "0%"), + updateStatusAvailable(false), + ) } diff --git a/pkg/backup/handlers/arango/backup/state_download_test.go b/pkg/backup/handlers/arango/backup/state_download_test.go index 107162f8a..dff1fafe2 100644 --- a/pkg/backup/handlers/arango/backup/state_download_test.go +++ b/pkg/backup/handlers/arango/backup/state_download_test.go @@ -56,24 +56,22 @@ func Test_State_Download_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateDownloading) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloading, false) require.NotNil(t, newObj.Status.Progress) progresses := mock.getProgressIDs() require.Len(t, progresses, 1) require.Equal(t, progresses[0], newObj.Status.Progress.JobID) - require.False(t, newObj.Status.Available) - require.Nil(t, newObj.Status.Backup) } // Check version func Test_State_Download_DownloadFailed(t *testing.T) { // Arrange - errorMsg := "download error" + error := newFatalErrorf("error") handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - downloadError: errorMsg, + downloadError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownload) @@ -93,23 +91,20 @@ func Test_State_Download_DownloadFailed(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloadError, false) require.Nil(t, newObj.Status.Progress) progresses := mock.getProgressIDs() require.Len(t, progresses, 0) - require.False(t, newObj.Status.Available) - require.Nil(t, newObj.Status.Backup) } func Test_State_Download_TemporaryDownloadFailed(t *testing.T) { // Arrange - errorMsg := "download error" - handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - isTemporaryError: true, - downloadError: errorMsg, + error := newTemporaryErrorf("error") + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + downloadError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownload) @@ -125,8 +120,15 @@ func Test_State_Download_TemporaryDownloadFailed(t *testing.T) { createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - err := handler.Handle(newItemFromBackup(operation.Update, obj)) + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) // Assert - compareTemporaryState(t, err, errorMsg, handler, obj) + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloadError, false) + + require.Nil(t, newObj.Status.Progress) + progresses := mock.getProgressIDs() + require.Len(t, progresses, 0) + + require.Nil(t, newObj.Status.Backup) } diff --git a/pkg/backup/handlers/arango/backup/state_downloaderror.go b/pkg/backup/handlers/arango/backup/state_downloaderror.go index cd5147035..ba0e1f115 100644 --- a/pkg/backup/handlers/arango/backup/state_downloaderror.go +++ b/pkg/backup/handlers/arango/backup/state_downloaderror.go @@ -32,17 +32,12 @@ const ( downloadDelay = time.Minute ) -func stateDownloadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateDownloadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { // Start again download if backup.Status.Time.Time.Add(downloadDelay).Before(time.Now()) { - return backupApi.ArangoBackupStatus{ - Available: false, - ArangoBackupState: newState(backupApi.ArangoBackupStatePending, "", nil), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStatePending, "")) } - return backupApi.ArangoBackupStatus{ - Available: false, - ArangoBackupState: backup.Status.ArangoBackupState, - }, nil + return wrapUpdateStatus(backup) } diff --git a/pkg/backup/handlers/arango/backup/state_downloaderror_test.go b/pkg/backup/handlers/arango/backup/state_downloaderror_test.go index e02575c8f..e0c5d084d 100644 --- a/pkg/backup/handlers/arango/backup/state_downloaderror_test.go +++ b/pkg/backup/handlers/arango/backup/state_downloaderror_test.go @@ -55,9 +55,7 @@ func Test_State_DownloadError_Reschedule(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStatePending) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStatePending, false) require.Nil(t, newObj.Status.Backup) } @@ -86,9 +84,8 @@ func Test_State_DownloadError_Wait(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateDownloadError) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloadError, false) - require.False(t, newObj.Status.Available) require.Equal(t, "message", newObj.Status.Message) require.Nil(t, newObj.Status.Backup) diff --git a/pkg/backup/handlers/arango/backup/state_downloading.go b/pkg/backup/handlers/arango/backup/state_downloading.go index e5f2d97dc..dd54cee0f 100644 --- a/pkg/backup/handlers/arango/backup/state_downloading.go +++ b/pkg/backup/handlers/arango/backup/state_downloading.go @@ -26,72 +26,79 @@ import ( "fmt" "github.com/arangodb/kube-arangodb/pkg/util" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateDownloadingHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateDownloadingHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { deployment, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } client, err := h.arangoClientFactory(deployment, backup) if err != nil { - return backupApi.ArangoBackupStatus{}, NewTemporaryError("unable to create client: %s", err.Error()) + return nil, newTemporaryError(err) } if backup.Status.Progress == nil { - return createFailedState(fmt.Errorf("backup progress details are missing"), backup.Status), nil + return nil, newFatalErrorf("backup progress details are missing") } if backup.Spec.Download == nil { - return createFailedState(fmt.Errorf("missing field .spec.download"), backup.Status), nil + return nil, newFatalErrorf("missing field .spec.download") } if backup.Spec.Download.ID == "" { - return createFailedState(fmt.Errorf("missing field .spec.download.id"), backup.Status), nil + return nil, newFatalErrorf("missing field .spec.download.id") } details, err := client.Progress(driver.BackupTransferJobID(backup.Status.Progress.JobID)) if err != nil { - return backup.Status, nil + if driver.IsNotFound(err) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloadError, + "job with id %s does not exist anymore", backup.Status.Progress.JobID), + cleanStatusJob(), + ) + } + + return nil, newTemporaryError(err) } if details.Failed { - return backupApi.ArangoBackupStatus{ - Available: false, - ArangoBackupState: newState(backupApi.ArangoBackupStateDownloadError, - fmt.Sprintf("Download failed with error: %s", details.FailMessage), nil), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloadError, + "Download failed with error: %s", details.FailMessage), + cleanStatusJob(), + ) } if details.Completed { backupMeta, err := client.Get(driver.BackupID(backup.Spec.Download.ID)) if err != nil { - return switchTemporaryError(err, backup.Status) + if driver.IsNotFound(err) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloadError, + "backup is not present after download"), + cleanStatusJob(), + ) + } + + return nil, newTemporaryError(err) } - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - Downloaded: util.NewBool(true), - }, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady,""), + updateStatusAvailable(true), + updateStatusBackup(backupMeta), + updateStatusBackupDownload(util.NewBool(true)), + cleanStatusJob(), + ) } - return backupApi.ArangoBackupStatus{ - Available: false, - ArangoBackupState: newState(backupApi.ArangoBackupStateDownloading, "", - &backupApi.ArangoBackupProgress{ - JobID: backup.Status.Progress.JobID, - Progress: fmt.Sprintf("%d%%", details.Progress), - }), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownloading,""), + updateStatusJob(backup.Status.Progress.JobID, fmt.Sprintf("%d%%", details.Progress)), + ) } diff --git a/pkg/backup/handlers/arango/backup/state_downloading_test.go b/pkg/backup/handlers/arango/backup/state_downloading_test.go index 6789e5614..a54370c12 100644 --- a/pkg/backup/handlers/arango/backup/state_downloading_test.go +++ b/pkg/backup/handlers/arango/backup/state_downloading_test.go @@ -24,12 +24,12 @@ package backup import ( "fmt" + "github.com/arangodb/go-driver" "testing" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" "github.com/arangodb/kube-arangodb/pkg/backup/operator/operation" "github.com/stretchr/testify/require" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( @@ -47,17 +47,16 @@ func Test_State_Downloading_Success(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownloading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) progress, err := mock.Download(backupMeta.ID) require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ @@ -79,16 +78,15 @@ func Test_State_Downloading_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateDownloading, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloading, false) + require.NotNil(t, newObj.Status.Progress) require.Equal(t, fmt.Sprintf("%d%%", 0), newObj.Status.Progress.Progress) require.Equal(t, obj.Status.Progress.JobID, newObj.Status.Progress.JobID) - - require.False(t, newObj.Status.Available) }) t.Run("Restore percent after update", func(t *testing.T) { p := 55 - mock.progresses[progress] = ArangoBackupProgress{ + mock.state.progresses[progress] = ArangoBackupProgress{ Progress: p, } @@ -96,15 +94,14 @@ func Test_State_Downloading_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateDownloading, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloading, false) + require.NotNil(t, newObj.Status.Progress) require.Equal(t, fmt.Sprintf("%d%%", p), newObj.Status.Progress.Progress) require.Equal(t, string(progress), newObj.Status.Progress.JobID) - - require.False(t, newObj.Status.Available) }) t.Run("Finished", func(t *testing.T) { - mock.progresses[progress] = ArangoBackupProgress{ + mock.state.progresses[progress] = ArangoBackupProgress{ Completed: true, } @@ -112,7 +109,7 @@ func Test_State_Downloading_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) require.Nil(t, newObj.Status.Progress) require.True(t, newObj.Status.Available) @@ -127,23 +124,22 @@ func Test_State_Downloading_FailedDownload(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownloading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) progress, err := mock.Download(backupMeta.ID) require.NoError(t, err) errorMsg := errorString - mock.progresses[progress] = ArangoBackupProgress{ + mock.state.progresses[progress] = ArangoBackupProgress{ Failed: true, FailMessage: errorMsg, } - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ @@ -164,33 +160,30 @@ func Test_State_Downloading_FailedDownload(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateDownloadError, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloadError, false) require.Equal(t, fmt.Sprintf("Download failed with error: %s", errorMsg), newObj.Status.Message) require.Nil(t, newObj.Status.Progress) - - require.False(t, newObj.Status.Available) } func Test_State_Downloading_FailedProgress(t *testing.T) { // Arrange - errorMsg := progressError + error := newFatalErrorf("error") handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - progressError: errorMsg, + progressError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownloading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) progress, err := mock.Download(backupMeta.ID) require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ @@ -207,37 +200,32 @@ func Test_State_Downloading_FailedProgress(t *testing.T) { createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + require.EqualError(t, handler.Handle(newItemFromBackup(operation.Update, obj)), error.Error()) // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateDownloading, newObj.Status.State) - require.NotNil(t, newObj.Status.Progress) - - require.False(t, newObj.Status.Available) + require.Equal(t, obj.Status, newObj.Status) } func Test_State_Downloading_TemporaryFailedProgress(t *testing.T) { // Arrange - errorMsg := progressError + error := newTemporaryErrorf("error") handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - isTemporaryError: true, - progressError: errorMsg, + progressError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownloading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) progress, err := mock.Download(backupMeta.ID) require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ @@ -254,12 +242,55 @@ func Test_State_Downloading_TemporaryFailedProgress(t *testing.T) { createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - err = handler.Handle(newItemFromBackup(operation.Update, obj)) + require.EqualError(t, handler.Handle(newItemFromBackup(operation.Update, obj)), error.Error()) // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateDownloading, newObj.Status.State) - require.NotNil(t, newObj.Status.Progress) - - require.False(t, newObj.Status.Available) + require.Equal(t, obj.Status, newObj.Status) } + +func Test_State_Downloading_NotFoundProgress(t *testing.T) { + // Arrange + error := driver.ArangoError{ + Code: 404, + } + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + progressError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateDownloading) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + progress, err := mock.Download(backupMeta.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ + ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ + RepositoryURL: "S3 URL", + }, + ID: string(backupMeta.ID), + } + + obj.Status.Progress = &backupApi.ArangoBackupProgress{ + JobID: string(progress), + } + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownloadError, false) + require.Equal(t, fmt.Sprintf("job with id %s does not exist anymore", progress), newObj.Status.Message) + require.Nil(t, newObj.Status.Progress) +} \ No newline at end of file diff --git a/pkg/backup/handlers/arango/backup/state_failed.go b/pkg/backup/handlers/arango/backup/state_failed.go index 2926f233b..ea0b12a2e 100644 --- a/pkg/backup/handlers/arango/backup/state_failed.go +++ b/pkg/backup/handlers/arango/backup/state_failed.go @@ -26,6 +26,6 @@ import ( backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateFailedHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { - return backup.Status, nil +func stateFailedHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + return wrapUpdateStatus(backup) } diff --git a/pkg/backup/handlers/arango/backup/state_none.go b/pkg/backup/handlers/arango/backup/state_none.go index 3790d77b5..32f9bdf54 100644 --- a/pkg/backup/handlers/arango/backup/state_none.go +++ b/pkg/backup/handlers/arango/backup/state_none.go @@ -26,8 +26,7 @@ import ( backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateNoneHandler(*handler, *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { - return backupApi.ArangoBackupStatus{ - ArangoBackupState: newState(backupApi.ArangoBackupStatePending, "", nil), - }, nil +func stateNoneHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStatePending, "")) } diff --git a/pkg/backup/handlers/arango/backup/state_none_test.go b/pkg/backup/handlers/arango/backup/state_none_test.go index 1ed787bb8..d0eac3d1e 100644 --- a/pkg/backup/handlers/arango/backup/state_none_test.go +++ b/pkg/backup/handlers/arango/backup/state_none_test.go @@ -42,7 +42,5 @@ func Test_State_None_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStatePending) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStatePending, false) } diff --git a/pkg/backup/handlers/arango/backup/state_pending.go b/pkg/backup/handlers/arango/backup/state_pending.go index 672788cc3..e6352eaf6 100644 --- a/pkg/backup/handlers/arango/backup/state_pending.go +++ b/pkg/backup/handlers/arango/backup/state_pending.go @@ -26,24 +26,22 @@ import ( backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func statePendingHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func statePendingHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { _, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } running, err := isBackupRunning(backup, h.client.BackupV1alpha().ArangoBackups(backup.Namespace)) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } if running { - return backupApi.ArangoBackupStatus{ - ArangoBackupState: newState(backupApi.ArangoBackupStatePending, "backup already in process", nil), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStatePending, "backup already in process")) } - return backupApi.ArangoBackupStatus{ - ArangoBackupState: newState(backupApi.ArangoBackupStateScheduled, "", nil), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateScheduled, "")) } diff --git a/pkg/backup/handlers/arango/backup/state_pending_test.go b/pkg/backup/handlers/arango/backup/state_pending_test.go index 1c98f152d..5aa435f64 100644 --- a/pkg/backup/handlers/arango/backup/state_pending_test.go +++ b/pkg/backup/handlers/arango/backup/state_pending_test.go @@ -55,9 +55,11 @@ func Test_State_Pending_CheckNamespaceIsolation(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) + checkBackup(t, newObj, backupApi.ArangoBackupStateFailed, false) - require.Equal(t, newObj.Status.Message, createFailMessage(backupApi.ArangoBackupStatePending, fmt.Sprintf("%s \"%s\" not found", database.ArangoDeploymentCRDName, obj.Name))) + require.Equal(t, newObj.Status.Message, + createStateMessage(backupApi.ArangoBackupStatePending, backupApi.ArangoBackupStateFailed, + fmt.Sprintf("%s \"%s\" not found", database.ArangoDeploymentCRDName, obj.Name))) } func Test_State_Pending_OneBackupObject(t *testing.T) { @@ -74,9 +76,7 @@ func Test_State_Pending_OneBackupObject(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateScheduled) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateScheduled, false) } func Test_State_Pending_MultipleBackupObjectWithLimitation(t *testing.T) { @@ -97,9 +97,7 @@ func Test_State_Pending_MultipleBackupObjectWithLimitation(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateScheduled) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateScheduled, false) }) t.Run("Second backup object", func(t *testing.T) { @@ -107,7 +105,7 @@ func Test_State_Pending_MultipleBackupObjectWithLimitation(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj2) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStatePending) + checkBackup(t, newObj, backupApi.ArangoBackupStatePending, false) require.Equal(t, newObj.Status.Message, "backup already in process") }) } diff --git a/pkg/backup/handlers/arango/backup/state_ready.go b/pkg/backup/handlers/arango/backup/state_ready.go index 5ba31db48..0cb6c52f3 100644 --- a/pkg/backup/handlers/arango/backup/state_ready.go +++ b/pkg/backup/handlers/arango/backup/state_ready.go @@ -23,39 +23,45 @@ package backup import ( - "fmt" - "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateReadyHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateReadyHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { deployment, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } client, err := h.arangoClientFactory(deployment, backup) if err != nil { - return backupApi.ArangoBackupStatus{}, NewTemporaryError("unable to create client: %s", err.Error()) + return nil, newTemporaryError(err) } if backup.Status.Backup == nil { - return createFailedState(fmt.Errorf("backup details are missing"), backup.Status), nil + return nil, newFatalErrorf("missing field .status.backup") } - _, err = client.Get(driver.BackupID(backup.Status.Backup.ID)) + backupMeta, err := client.Get(driver.BackupID(backup.Status.Backup.ID)) if err != nil { if driver.IsNotFound(err) { - return backupApi.ArangoBackupStatus{ - ArangoBackupState: backupApi.ArangoBackupState{ - State: backupApi.ArangoBackupStateDeleted, - }, - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDeleted, ""), + updateStatusAvailable(false), + ) } - return backup.Status, nil + return wrapUpdateStatus(backup, + updateStatusAvailable(true), + ) + } + + if backupMeta.Available == false { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUnavailable, ""), + updateStatusBackup(backupMeta), + updateStatusAvailable(false), + ) } // Check if upload flag was specified later in runtime @@ -64,38 +70,35 @@ func stateReadyHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.Ar // Ensure that we can start upload process running, err := isBackupRunning(backup, h.client.BackupV1alpha().ArangoBackups(backup.Namespace)) if err != nil { - return backup.Status, nil + return nil, err } if running { - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "Upload process queued", nil), - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady, "Upload process queued"), + updateStatusBackup(backupMeta), + updateStatusAvailable(true), + ) } - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateUpload, "", nil), - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUpload, ""), + updateStatusBackup(backupMeta), + updateStatusAvailable(true), + ) } - // Remove old upload flag if backup.Spec.Upload == nil && backup.Status.Backup.Uploaded != nil { - newBackup := backup.Status.Backup.DeepCopy() - newBackup.Uploaded = nil - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: newBackup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady, ""), + updateStatusBackup(backupMeta), + updateStatusBackupUpload(nil), + updateStatusAvailable(true), + ) } - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusBackup(backupMeta), + updateStatusAvailable(true), + ) } diff --git a/pkg/backup/handlers/arango/backup/state_ready_test.go b/pkg/backup/handlers/arango/backup/state_ready_test.go index c2457a664..73cf441d2 100644 --- a/pkg/backup/handlers/arango/backup/state_ready_test.go +++ b/pkg/backup/handlers/arango/backup/state_ready_test.go @@ -23,17 +23,16 @@ package backup import ( + "github.com/arangodb/go-driver" "github.com/arangodb/kube-arangodb/pkg/util" + "k8s.io/apimachinery/pkg/util/uuid" "sync" "testing" - "k8s.io/apimachinery/pkg/util/uuid" - "github.com/arangodb/kube-arangodb/pkg/backup/operator/operation" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" "github.com/stretchr/testify/require" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_State_Ready_Common(t *testing.T) { @@ -47,15 +46,13 @@ func Test_State_Ready_Success(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) // Act createArangoDeployment(t, handler, deployment) @@ -65,28 +62,27 @@ func Test_State_Ready_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, obj.Status, newObj.Status) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) } -func Test_State_Ready_GetFailed(t *testing.T) { +func Test_State_Ready_Unavailable(t *testing.T) { // Arrange - errorMsg := "get error" - handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - getError: errorMsg, - }) + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + backupMeta.Available = false + + mock.state.backups[createResponse.ID] = backupMeta // Act createArangoDeployment(t, handler, deployment) @@ -96,40 +92,126 @@ func Test_State_Ready_GetFailed(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) + compareBackupMeta(t, backupMeta, newObj) +} + +func Test_State_Ready_Success_Update(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + t.Run("First iteration", func(t *testing.T) { + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) + }) + + t.Run("Second iteration", func(t *testing.T) { + backupMeta.SizeInBytes = 123 + mock.state.backups[backupMeta.ID] = backupMeta + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) + require.Equal(t, uint64(123), newObj.Status.Backup.SizeInBytes) + }) + + t.Run("Do nothing", func(t *testing.T) { + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) + }) } func Test_State_Ready_TemporaryGetFailed(t *testing.T) { // Arrange - errorMsg := "get error" - handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - isTemporaryError: true, - getError: errorMsg, + error := newTemporaryErrorf("error") + handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) - backupMeta, err := mock.Create() - require.NoError(t, err) + obj.Status.Backup = &backupApi.ArangoBackupDetails{} + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) +} - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), +func Test_State_Ready_FatalGetFailed(t *testing.T) { + // Arrange + error := newFatalErrorf("error") + handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{} + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) +} + +func Test_State_Ready_MissingBackup(t *testing.T) { + // Arrange + error := driver.ArangoError{ + Code: 404, } - obj.Status.Available = true + handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{} // Act createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - err = handler.Handle(newItemFromBackup(operation.Update, obj)) + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateDeleted, false) } func Test_State_Ready_Upload(t *testing.T) { @@ -141,15 +223,13 @@ func Test_State_Ready_Upload(t *testing.T) { RepositoryURL: "Any", } - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) // Act createArangoDeployment(t, handler, deployment) @@ -159,8 +239,8 @@ func Test_State_Ready_Upload(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUpload, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateUpload, true) + compareBackupMeta(t, backupMeta, newObj) } func Test_State_Ready_DownloadDoNothing(t *testing.T) { @@ -175,15 +255,15 @@ func Test_State_Ready_DownloadDoNothing(t *testing.T) { ID: "some", } - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, &backupApi.ArangoBackupDetails{ + Downloaded: util.NewBool(true), + }) // Act createArangoDeployment(t, handler, deployment) @@ -193,8 +273,10 @@ func Test_State_Ready_DownloadDoNothing(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) + require.NotNil(t, newObj.Status.Backup.Downloaded) + require.True(t, *newObj.Status.Backup.Downloaded) } func Test_State_Ready_DoUploadDownloadedBackup(t *testing.T) { @@ -206,17 +288,16 @@ func Test_State_Ready_DoUploadDownloadedBackup(t *testing.T) { RepositoryURL: "Any", } - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - Downloaded: util.NewBool(true), - Uploaded: util.NewBool(false), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, &backupApi.ArangoBackupDetails{ + Downloaded: util.NewBool(true), + Uploaded: util.NewBool(false), + }) // Act createArangoDeployment(t, handler, deployment) @@ -226,8 +307,9 @@ func Test_State_Ready_DoUploadDownloadedBackup(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUpload, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateUpload, true) + require.NotNil(t, newObj.Status.Backup.Downloaded) + require.True(t, *newObj.Status.Backup.Downloaded) } func Test_State_Ready_DoNotReUploadBackup(t *testing.T) { @@ -239,16 +321,15 @@ func Test_State_Ready_DoNotReUploadBackup(t *testing.T) { RepositoryURL: "Any", } - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - Uploaded: util.NewBool(true), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, &backupApi.ArangoBackupDetails{ + Uploaded: util.NewBool(true), + }) // Act createArangoDeployment(t, handler, deployment) @@ -258,8 +339,10 @@ func Test_State_Ready_DoNotReUploadBackup(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) + require.NotNil(t, newObj.Status.Backup.Uploaded) + require.True(t, *newObj.Status.Backup.Uploaded) } func Test_State_Ready_RemoveUploadedFlag(t *testing.T) { @@ -268,16 +351,15 @@ func Test_State_Ready_RemoveUploadedFlag(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateReady) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - Uploaded: util.NewBool(true), - } - obj.Status.Available = true + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, &backupApi.ArangoBackupDetails{ + Uploaded: util.NewBool(true), + }) // Act createArangoDeployment(t, handler, deployment) @@ -287,8 +369,7 @@ func Test_State_Ready_RemoveUploadedFlag(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) require.Nil(t, newObj.Status.Backup.Uploaded) } @@ -302,16 +383,16 @@ func Test_State_Ready_KeepPendingWithForcedRunning(t *testing.T) { size := 128 objects := make([]*backupApi.ArangoBackup, size) for id := range objects { - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) obj := newArangoBackup(name, name, string(uuid.NewUUID()), backupApi.ArangoBackupStateReady) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + obj.Spec.Upload = &backupApi.ArangoBackupSpecOperation{ RepositoryURL: "s3://test", } @@ -362,7 +443,10 @@ func Test_State_Ready_KeepPendingWithForcedRunningSameId(t *testing.T) { name := string(uuid.NewUUID()) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) deployment := newArangoDeployment(name, name) @@ -372,11 +456,7 @@ func Test_State_Ready_KeepPendingWithForcedRunningSameId(t *testing.T) { obj := newArangoBackup(name, name, string(uuid.NewUUID()), backupApi.ArangoBackupStateReady) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Spec.Upload = &backupApi.ArangoBackupSpecOperation{ RepositoryURL: "s3://test", } diff --git a/pkg/backup/handlers/arango/backup/state_scheduled.go b/pkg/backup/handlers/arango/backup/state_scheduled.go index 34af3d41b..3aaf2fa03 100644 --- a/pkg/backup/handlers/arango/backup/state_scheduled.go +++ b/pkg/backup/handlers/arango/backup/state_scheduled.go @@ -26,19 +26,18 @@ import ( backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateScheduledHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateScheduledHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + // If unable to get ArangoDeployment go into Failed state _, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } if backup.Spec.Download != nil { - return backupApi.ArangoBackupStatus{ - ArangoBackupState: newState(backupApi.ArangoBackupStateDownload, "", nil), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDownload, "")) } - return backupApi.ArangoBackupStatus{ - ArangoBackupState: newState(backupApi.ArangoBackupStateCreate, "", nil), - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateCreate, "")) } diff --git a/pkg/backup/handlers/arango/backup/state_scheduled_test.go b/pkg/backup/handlers/arango/backup/state_scheduled_test.go index 22afcef48..333e7bc6f 100644 --- a/pkg/backup/handlers/arango/backup/state_scheduled_test.go +++ b/pkg/backup/handlers/arango/backup/state_scheduled_test.go @@ -56,9 +56,7 @@ func Test_State_Scheduled_Download(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateDownload) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateDownload, false) } func Test_State_Scheduled_Create(t *testing.T) { @@ -75,9 +73,7 @@ func Test_State_Scheduled_Create(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreate) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateCreate, false) } func Test_State_Scheduled_Upload(t *testing.T) { @@ -98,7 +94,5 @@ func Test_State_Scheduled_Upload(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateCreate) - - require.False(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateCreate, false) } diff --git a/pkg/backup/handlers/arango/backup/state_unavailable.go b/pkg/backup/handlers/arango/backup/state_unavailable.go new file mode 100644 index 000000000..e7656bb82 --- /dev/null +++ b/pkg/backup/handlers/arango/backup/state_unavailable.go @@ -0,0 +1,72 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package backup + +import ( + "github.com/arangodb/go-driver" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" +) + +func stateUnavailableHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + deployment, err := h.getArangoDeploymentObject(backup) + if err != nil { + return nil, err + } + + client, err := h.arangoClientFactory(deployment, backup) + if err != nil { + return nil, newTemporaryError(err) + } + + if backup.Status.Backup == nil { + return nil, newFatalErrorf("missing field .status.backup") + } + + backupMeta, err := client.Get(driver.BackupID(backup.Status.Backup.ID)) + if err != nil { + if driver.IsNotFound(err) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDeleted, ""), + updateStatusAvailable(false), + ) + } + + return wrapUpdateStatus(backup, + updateStatusAvailable(false), + ) + } + + if backupMeta.Available == false { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUnavailable, ""), + updateStatusBackup(backupMeta), + updateStatusAvailable(false), + ) + } + + return wrapUpdateStatus(backup, + updateStatusBackup(backupMeta), + updateStatusState(backupApi.ArangoBackupStateReady, ""), + updateStatusAvailable(true), + ) +} diff --git a/pkg/backup/handlers/arango/backup/state_unavailable_test.go b/pkg/backup/handlers/arango/backup/state_unavailable_test.go new file mode 100644 index 000000000..b076a9f37 --- /dev/null +++ b/pkg/backup/handlers/arango/backup/state_unavailable_test.go @@ -0,0 +1,215 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package backup + +import ( + "github.com/arangodb/go-driver" + "github.com/arangodb/kube-arangodb/pkg/backup/operator/operation" + "github.com/stretchr/testify/require" + "testing" + + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" +) + +func Test_State_Unavailable_Common(t *testing.T) { + wrapperUndefinedDeployment(t, backupApi.ArangoBackupStateUnavailable) + wrapperConnectionIssues(t, backupApi.ArangoBackupStateUnavailable) +} + +func Test_State_Unavailable_Success(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUnavailable) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) + compareBackupMeta(t, backupMeta, newObj) +} + +func Test_State_Unavailable_Keep(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUnavailable) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + backupMeta.Available = false + + mock.state.backups[createResponse.ID] = backupMeta + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) + compareBackupMeta(t, backupMeta, newObj) +} + +func Test_State_Unavailable_Update(t *testing.T) { + // Arrange + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{}) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUnavailable) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + backupMeta.Available = false + + mock.state.backups[createResponse.ID] = backupMeta + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + t.Run("First iteration", func(t *testing.T) { + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) + compareBackupMeta(t, backupMeta, newObj) + }) + + t.Run("Second iteration", func(t *testing.T) { + backupMeta.SizeInBytes = 123 + mock.state.backups[backupMeta.ID] = backupMeta + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) + compareBackupMeta(t, backupMeta, newObj) + require.Equal(t, uint64(123), newObj.Status.Backup.SizeInBytes) + }) + + t.Run("Do nothing", func(t *testing.T) { + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) + compareBackupMeta(t, backupMeta, newObj) + }) +} + +func Test_State_Unavailable_TemporaryGetFailed(t *testing.T) { + // Arrange + error := newTemporaryErrorf("error") + handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUnavailable) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{} + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) +} + +func Test_State_Unavailable_FatalGetFailed(t *testing.T) { + // Arrange + error := newFatalErrorf("error") + handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUnavailable) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{} + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUnavailable, false) +} + +func Test_State_Unavailable_MissingBackup(t *testing.T) { + // Arrange + error := driver.ArangoError{ + Code: 404, + } + handler, _ := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUnavailable) + + obj.Status.Backup = &backupApi.ArangoBackupDetails{} + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateDeleted, false) +} diff --git a/pkg/backup/handlers/arango/backup/state_upload.go b/pkg/backup/handlers/arango/backup/state_upload.go index 0ed6e14cc..9bcba6c27 100644 --- a/pkg/backup/handlers/arango/backup/state_upload.go +++ b/pkg/backup/handlers/arango/backup/state_upload.go @@ -23,44 +23,51 @@ package backup import ( - "fmt" - "github.com/arangodb/go-driver" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateUploadHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateUploadHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { deployment, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } client, err := h.arangoClientFactory(deployment, backup) if err != nil { - return backupApi.ArangoBackupStatus{}, NewTemporaryError("unable to create client: %s", err.Error()) + return nil, newTemporaryError(err) } if backup.Status.Backup == nil { - return createFailedState(fmt.Errorf("backup details are missing"), backup.Status), nil + return nil, newFatalErrorf("missing field .status.backup") } meta, err := client.Get(driver.BackupID(backup.Status.Backup.ID)) if err != nil { - return switchTemporaryError(err, backup.Status) + if driver.IsNotFound(err) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateDeleted, ""), + updateStatusAvailable(false), + ) + } + + return nil, newTemporaryError(err) } jobID, err := client.Upload(meta.ID) if err != nil { - return switchTemporaryError(err, backup.Status) + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUploadError, + "Upload failed with error: %s", err.Error()), + cleanStatusJob(), + updateStatusBackupUpload(nil), + updateStatusAvailable(true), + ) } - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateUploading, "", - &backupApi.ArangoBackupProgress{ - JobID: string(jobID), - Progress: "0%", - }), - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUploading, ""), + updateStatusJob(string(jobID), "0%"), + updateStatusAvailable(true), + ) } diff --git a/pkg/backup/handlers/arango/backup/state_upload_test.go b/pkg/backup/handlers/arango/backup/state_upload_test.go index 4aa70e139..ff24323b8 100644 --- a/pkg/backup/handlers/arango/backup/state_upload_test.go +++ b/pkg/backup/handlers/arango/backup/state_upload_test.go @@ -23,18 +23,13 @@ package backup import ( + "github.com/arangodb/go-driver" "testing" "github.com/arangodb/kube-arangodb/pkg/backup/operator/operation" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" "github.com/stretchr/testify/require" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -const ( - getError = "get error" - uploadError = "upload error" ) func Test_State_Upload_Common(t *testing.T) { @@ -48,14 +43,13 @@ func Test_State_Upload_Success(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) // Act createArangoDeployment(t, handler, deployment) @@ -65,110 +59,149 @@ func Test_State_Upload_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateUploading) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploading, true) require.NotNil(t, newObj.Status.Progress) progresses := mock.getProgressIDs() require.Len(t, progresses, 1) require.Equal(t, progresses[0], newObj.Status.Progress.JobID) - require.True(t, newObj.Status.Available) - - require.NotNil(t, newObj.Status.Backup) - require.Equal(t, string(backupMeta.ID), newObj.Status.Backup.ID) - require.Equal(t, backupMeta.Version, newObj.Status.Backup.Version) + compareBackupMeta(t, backupMeta, obj) } -func Test_State_Upload_Failed(t *testing.T) { +func Test_State_Upload_TemporaryGetFailed(t *testing.T) { // Arrange - checks := map[string]mockErrorsArangoClientBackup{ - "get": { - getError: getError, - }, - "upload": { - uploadError: uploadError, - }, - } + error := newTemporaryErrorf("error") + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) - for name, c := range checks { - t.Run(name, func(t *testing.T) { - // Arrange - handler, mock := newErrorsFakeHandler(c) + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) - obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) + createResponse, err := mock.Create() + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{ + ID: createResponse.ID, + }, nil) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.EqualError(t, handler.Handle(newItemFromBackup(operation.Update, obj)), error.Error()) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + require.Equal(t, obj.Status, newObj.Status) +} - backupMeta, err := mock.Create() - require.NoError(t, err) +func Test_State_Upload_FatalGetFailed(t *testing.T) { + // Arrange + error := newFatalErrorf("error") + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) - // Act - createArangoDeployment(t, handler, deployment) - createArangoBackup(t, handler, obj) + createResponse, err := mock.Create() + require.NoError(t, err) - require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{ + ID: createResponse.ID, + }, nil) - // Assert - newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateFailed) + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) - require.Nil(t, newObj.Status.Progress) - progresses := mock.getProgressIDs() - require.Len(t, progresses, 0) + require.EqualError(t, handler.Handle(newItemFromBackup(operation.Update, obj)), error.Error()) - require.False(t, newObj.Status.Available) + // Assert + newObj := refreshArangoBackup(t, handler, obj) + require.Equal(t, obj.Status, newObj.Status) +} - require.NotNil(t, newObj.Status.Backup) - require.Equal(t, string(backupMeta.ID), newObj.Status.Backup.ID) - require.Equal(t, backupMeta.Version, newObj.Status.Backup.Version) - }) +func Test_State_Upload_BackupMissing(t *testing.T) { + // Arrange + error := driver.ArangoError{ + Code: 404, } + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + getError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) + + createResponse, err := mock.Create() + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{ + ID: createResponse.ID, + }, nil) + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateDeleted, false) } -func Test_State_Upload_TemporaryFailed(t *testing.T) { +func Test_State_Upload_TemporaryUploadFailed(t *testing.T) { // Arrange - checks := map[string]mockErrorsArangoClientBackup{ - "get": { - getError: getError, + error := newTemporaryErrorf("error") + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + uploadError: error, + }) - isTemporaryError: true, - }, - "upload": { - uploadError: uploadError, + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) - isTemporaryError: true, - }, - } + createResponse, err := mock.Create() + require.NoError(t, err) - for name, c := range checks { - t.Run(name, func(t *testing.T) { - // Arrange - handler, mock := newErrorsFakeHandler(c) + obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{ + ID: createResponse.ID, + }, nil) - obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) - backupMeta, err := mock.Create() - require.NoError(t, err) + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true) +} + +func Test_State_Upload_FatalUploadFailed(t *testing.T) { + // Arrange + error := newFatalErrorf("error") + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + uploadError: error, + }) - // Act - createArangoDeployment(t, handler, deployment) - createArangoBackup(t, handler, obj) + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUpload) + + createResponse, err := mock.Create() + require.NoError(t, err) - err = handler.Handle(newItemFromBackup(operation.Update, obj)) + obj.Status.Backup = createBackupFromMeta(driver.BackupMeta{ + ID: createResponse.ID, + }, nil) - // Assert - require.Error(t, err) - require.True(t, IsTemporaryError(err)) - }) - } + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true) } diff --git a/pkg/backup/handlers/arango/backup/state_uploaderror.go b/pkg/backup/handlers/arango/backup/state_uploaderror.go index cb80c621e..501fa3a9b 100644 --- a/pkg/backup/handlers/arango/backup/state_uploaderror.go +++ b/pkg/backup/handlers/arango/backup/state_uploaderror.go @@ -32,28 +32,14 @@ const ( uploadDelay = time.Minute ) -func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { - // After upload removal go into Ready status - if backup.Spec.Upload == nil { - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: backup.Status.Backup.DeepCopy(), - }, nil +func stateUploadErrorHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { + if backup.Spec.Upload == nil || backup.Status.Time.Time.Add(uploadDelay).Before(time.Now()) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady, ""), + cleanStatusJob(), + updateStatusAvailable(true)) } - // Start again upload - if backup.Status.Time.Time.Add(uploadDelay).Before(time.Now()) { - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: backup.Status.Backup.DeepCopy(), - }, nil - } - - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: backup.Status.ArangoBackupState, - Backup: backup.Status.Backup.DeepCopy(), - }, nil + return wrapUpdateStatus(backup, + updateStatusAvailable(true)) } diff --git a/pkg/backup/handlers/arango/backup/state_uploaderror_test.go b/pkg/backup/handlers/arango/backup/state_uploaderror_test.go index 118ebe36d..4a27cd6b5 100644 --- a/pkg/backup/handlers/arango/backup/state_uploaderror_test.go +++ b/pkg/backup/handlers/arango/backup/state_uploaderror_test.go @@ -103,9 +103,8 @@ func Test_State_UploadError_Wait(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateUploadError) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true) - require.True(t, newObj.Status.Available) require.Equal(t, "message", newObj.Status.Message) require.NotNil(t, newObj.Status.Backup) @@ -137,9 +136,7 @@ func Test_State_UploadError_BackToReady(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, newObj.Status.State, backupApi.ArangoBackupStateReady) - - require.True(t, newObj.Status.Available) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) require.NotNil(t, newObj.Status.Backup) require.Equal(t, obj.Status.Backup, newObj.Status.Backup) diff --git a/pkg/backup/handlers/arango/backup/state_uploading.go b/pkg/backup/handlers/arango/backup/state_uploading.go index 9cace76af..e1338d73d 100644 --- a/pkg/backup/handlers/arango/backup/state_uploading.go +++ b/pkg/backup/handlers/arango/backup/state_uploading.go @@ -30,61 +30,59 @@ import ( backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" ) -func stateUploadingHandler(h *handler, backup *backupApi.ArangoBackup) (backupApi.ArangoBackupStatus, error) { +func stateUploadingHandler(h *handler, backup *backupApi.ArangoBackup) (*backupApi.ArangoBackupStatus, error) { deployment, err := h.getArangoDeploymentObject(backup) if err != nil { - return createFailedState(err, backup.Status), nil + return nil, err } client, err := h.arangoClientFactory(deployment, backup) if err != nil { - return backupApi.ArangoBackupStatus{}, NewTemporaryError("unable to create client: %s", err.Error()) + return nil, newTemporaryError(err) } if backup.Status.Backup == nil { - return createFailedState(fmt.Errorf("backup details are missing"), backup.Status), nil + return nil, newFatalErrorf("missing field .status.backup") } if backup.Status.Progress == nil { - return createFailedState(fmt.Errorf("backup progress details are missing"), backup.Status), nil + return nil, newFatalErrorf("missing field .status.progress") } details, err := client.Progress(driver.BackupTransferJobID(backup.Status.Progress.JobID)) if err != nil { if driver.IsNotFound(err) { - return switchTemporaryError(err, backup.Status) + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUploadError, + "job with id %s does not exist anymore", backup.Status.Progress.JobID), + cleanStatusJob(), + updateStatusAvailable(true), + ) } - return backup.Status, nil + return nil, newTemporaryError(err) } if details.Failed { - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateUploadError, - fmt.Sprintf("Upload failed with error: %s", details.FailMessage), nil), - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUploadError, + "Upload failed with error: %s", details.FailMessage), + cleanStatusJob(), + updateStatusAvailable(true), + ) } if details.Completed { - newDetails := backup.Status.Backup.DeepCopy() - - newDetails.Uploaded = util.NewBool(true) - - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateReady, "", nil), - Backup: newDetails, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateReady, ""), + cleanStatusJob(), + updateStatusBackupUpload(util.NewBool(true)), + updateStatusAvailable(true), + ) } - return backupApi.ArangoBackupStatus{ - Available: true, - ArangoBackupState: newState(backupApi.ArangoBackupStateUploading, "", - &backupApi.ArangoBackupProgress{ - JobID: backup.Status.Progress.JobID, - Progress: fmt.Sprintf("%d%%", details.Progress), - }), - Backup: backup.Status.Backup, - }, nil + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateUploading,""), + updateStatusAvailable(true), + updateStatusJob(backup.Status.Progress.JobID, fmt.Sprintf("%d%%", details.Progress)), + ) } diff --git a/pkg/backup/handlers/arango/backup/state_uploading_test.go b/pkg/backup/handlers/arango/backup/state_uploading_test.go index 26a4fd454..7dee70291 100644 --- a/pkg/backup/handlers/arango/backup/state_uploading_test.go +++ b/pkg/backup/handlers/arango/backup/state_uploading_test.go @@ -24,13 +24,13 @@ package backup import ( "fmt" + "github.com/arangodb/go-driver" "testing" "github.com/arangodb/kube-arangodb/pkg/backup/operator/operation" backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" "github.com/stretchr/testify/require" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" ) func Test_State_Uploading_Common(t *testing.T) { @@ -45,17 +45,16 @@ func Test_State_Uploading_Success(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateUploading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) progress, err := mock.Upload(backupMeta.ID) require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Status.Progress = &backupApi.ArangoBackupProgress{ JobID: string(progress), @@ -70,16 +69,14 @@ func Test_State_Uploading_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUploading, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploading, true) require.Equal(t, fmt.Sprintf("%d%%", 0), newObj.Status.Progress.Progress) require.Equal(t, obj.Status.Progress.JobID, newObj.Status.Progress.JobID) - - require.True(t, newObj.Status.Available) }) t.Run("Restore percent after update", func(t *testing.T) { p := 55 - mock.progresses[progress] = ArangoBackupProgress{ + mock.state.progresses[progress] = ArangoBackupProgress{ Progress: p, } @@ -87,15 +84,13 @@ func Test_State_Uploading_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUploading, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploading, true) require.Equal(t, fmt.Sprintf("%d%%", p), newObj.Status.Progress.Progress) require.Equal(t, string(progress), newObj.Status.Progress.JobID) - - require.True(t, newObj.Status.Available) }) t.Run("Finished", func(t *testing.T) { - mock.progresses[progress] = ArangoBackupProgress{ + mock.state.progresses[progress] = ArangoBackupProgress{ Completed: true, } @@ -103,10 +98,9 @@ func Test_State_Uploading_Success(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateReady, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateReady, true) require.Nil(t, newObj.Status.Progress) - require.True(t, newObj.Status.Available) require.NotNil(t, newObj.Status.Backup.Uploaded) require.True(t, *newObj.Status.Backup.Uploaded) }) @@ -118,23 +112,22 @@ func Test_State_Uploading_FailedUpload(t *testing.T) { obj, deployment := newObjectSet(backupApi.ArangoBackupStateUploading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) progress, err := mock.Upload(backupMeta.ID) require.NoError(t, err) errorMsg := errorString - mock.progresses[progress] = ArangoBackupProgress{ + mock.state.progresses[progress] = ArangoBackupProgress{ Failed: true, FailMessage: errorMsg, } - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) obj.Status.Progress = &backupApi.ArangoBackupProgress{ JobID: string(progress), @@ -148,35 +141,37 @@ func Test_State_Uploading_FailedUpload(t *testing.T) { // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUploadError, newObj.Status.State) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true) require.Equal(t, fmt.Sprintf("Upload failed with error: %s", errorMsg), newObj.Status.Message) require.Nil(t, newObj.Status.Progress) - - require.True(t, newObj.Status.Available) } -func Test_State_Uploading_FailedProgress(t *testing.T) { +func Test_StateUploading_FailedProgress(t *testing.T) { // Arrange - errorMsg := "progress error" + error := newFatalErrorf("error") handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - progressError: errorMsg, + progressError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateUploading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - progress, err := mock.Upload(backupMeta.ID) + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + progress, err := mock.Download(backupMeta.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) - obj.Status.Available = true + obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ + ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ + RepositoryURL: "S3 URL", + }, + ID: string(backupMeta.ID), + } obj.Status.Progress = &backupApi.ArangoBackupProgress{ JobID: string(progress), @@ -186,39 +181,39 @@ func Test_State_Uploading_FailedProgress(t *testing.T) { createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + require.EqualError(t, handler.Handle(newItemFromBackup(operation.Update, obj)), error.Error()) // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUploading, newObj.Status.State) - require.NotNil(t, newObj.Status.Progress) - - require.True(t, newObj.Status.Available) + require.Equal(t, obj.Status, newObj.Status) } func Test_State_Uploading_TemporaryFailedProgress(t *testing.T) { // Arrange - errorMsg := "progress error" + error := newTemporaryErrorf("error") handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ - isTemporaryError: true, - progressError: errorMsg, + progressError: error, }) obj, deployment := newObjectSet(backupApi.ArangoBackupStateUploading) - backupMeta, err := mock.Create() + createResponse, err := mock.Create() require.NoError(t, err) - progress, err := mock.Upload(backupMeta.ID) + backupMeta, err := mock.Get(createResponse.ID) require.NoError(t, err) - obj.Status.Backup = &backupApi.ArangoBackupDetails{ - ID: string(backupMeta.ID), - Version: backupMeta.Version, - CreationTimestamp: meta.Now(), - } + progress, err := mock.Download(backupMeta.ID) + require.NoError(t, err) - obj.Status.Available = true + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ + ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ + RepositoryURL: "S3 URL", + }, + ID: string(backupMeta.ID), + } obj.Status.Progress = &backupApi.ArangoBackupProgress{ JobID: string(progress), @@ -228,12 +223,55 @@ func Test_State_Uploading_TemporaryFailedProgress(t *testing.T) { createArangoDeployment(t, handler, deployment) createArangoBackup(t, handler, obj) - err = handler.Handle(newItemFromBackup(operation.Update, obj)) + require.EqualError(t, handler.Handle(newItemFromBackup(operation.Update, obj)), error.Error()) // Assert newObj := refreshArangoBackup(t, handler, obj) - require.Equal(t, backupApi.ArangoBackupStateUploading, newObj.Status.State) - require.NotNil(t, newObj.Status.Progress) + require.Equal(t, obj.Status, newObj.Status) +} - require.True(t, newObj.Status.Available) +func Test_State_Uploading_NotFoundProgress(t *testing.T) { + // Arrange + error := driver.ArangoError{ + Code: 404, + } + handler, mock := newErrorsFakeHandler(mockErrorsArangoClientBackup{ + progressError: error, + }) + + obj, deployment := newObjectSet(backupApi.ArangoBackupStateUploading) + + createResponse, err := mock.Create() + require.NoError(t, err) + + backupMeta, err := mock.Get(createResponse.ID) + require.NoError(t, err) + + progress, err := mock.Download(backupMeta.ID) + require.NoError(t, err) + + obj.Status.Backup = createBackupFromMeta(backupMeta, nil) + + obj.Spec.Download = &backupApi.ArangoBackupSpecDownload{ + ArangoBackupSpecOperation: backupApi.ArangoBackupSpecOperation{ + RepositoryURL: "S3 URL", + }, + ID: string(backupMeta.ID), + } + + obj.Status.Progress = &backupApi.ArangoBackupProgress{ + JobID: string(progress), + } + + // Act + createArangoDeployment(t, handler, deployment) + createArangoBackup(t, handler, obj) + + require.NoError(t, handler.Handle(newItemFromBackup(operation.Update, obj))) + + // Assert + newObj := refreshArangoBackup(t, handler, obj) + checkBackup(t, newObj, backupApi.ArangoBackupStateUploadError, true) + require.Equal(t, fmt.Sprintf("job with id %s does not exist anymore", progress), newObj.Status.Message) + require.Nil(t, newObj.Status.Progress) } diff --git a/pkg/backup/handlers/arango/backup/status.go b/pkg/backup/handlers/arango/backup/status.go new file mode 100644 index 000000000..51ca327a1 --- /dev/null +++ b/pkg/backup/handlers/arango/backup/status.go @@ -0,0 +1,148 @@ +// +// DISCLAIMER +// +// Copyright 2018 ArangoDB GmbH, Cologne, Germany +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Copyright holder is ArangoDB GmbH, Cologne, Germany +// +// Author Adam Janikowski +// + +package backup + +import ( + "fmt" + "github.com/arangodb/go-driver" + backupApi "github.com/arangodb/kube-arangodb/pkg/apis/backup/v1alpha" + "github.com/arangodb/kube-arangodb/pkg/backup/state" + "github.com/arangodb/kube-arangodb/pkg/util" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type updateStatusFunc func(status *backupApi.ArangoBackupStatus) + +func wrapUpdateStatus(backup *backupApi.ArangoBackup, update ... updateStatusFunc) (*backupApi.ArangoBackupStatus, error) { + return updateStatus(backup, update...), nil +} + +func updateStatus(backup *backupApi.ArangoBackup, update ... updateStatusFunc) *backupApi.ArangoBackupStatus { + s := backup.Status.DeepCopy() + + for _, u := range update { + u(s) + } + + return s +} + +func updateStatusState(state state.State, template string, a ... interface{}) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + if status.State != state { + status.Time = v1.Now() + } + status.State = state + status.Message = fmt.Sprintf(template, a...) + } +} + +func updateStatusAvailable(available bool) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + status.Available = available + } +} + +func updateStatusJob(id, progress string) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + status.Progress = &backupApi.ArangoBackupProgress{ + JobID: id, + Progress: progress, + } + } +} + +func updateStatusBackupUpload(uploaded *bool) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + if status.Backup != nil { + status.Backup.Uploaded = uploaded + } + } +} + +func updateStatusBackupImported(imported *bool) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + if status.Backup != nil { + status.Backup.Imported = imported + } + } +} + +func updateStatusBackupDownload(downloaded *bool) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + if status.Backup != nil { + status.Backup.Downloaded = downloaded + } + } +} + +func updateStatusBackup(backupMeta driver.BackupMeta) updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + status.Backup = createBackupFromMeta(backupMeta, status.Backup) + } +} + +func cleanStatusJob() updateStatusFunc { + return func(status *backupApi.ArangoBackupStatus) { + status.Progress = nil + } +} + +func setFailedState(backup *backupApi.ArangoBackup, err error) (*backupApi.ArangoBackupStatus, error) { + return wrapUpdateStatus(backup, + updateStatusState(backupApi.ArangoBackupStateFailed, createStateMessage(backup.Status.State, backupApi.ArangoBackupStateFailed, err.Error())), + updateStatusAvailable(false)) +} + +func createStateMessage(from, to state.State, message string) string { + return fmt.Sprintf("Transiting from %s to %s: %s", from, to, message) +} + +func switchTemporaryError(backup *backupApi.ArangoBackup, err error) (*backupApi.ArangoBackupStatus, error) { + if _, ok := err.(temporaryError); ok { + return nil, err + } + + return setFailedState(backup, err) +} + +func createBackupFromMeta(backupMeta driver.BackupMeta, old *backupApi.ArangoBackupDetails) *backupApi.ArangoBackupDetails { + var obj *backupApi.ArangoBackupDetails + + if old == nil { + obj = &backupApi.ArangoBackupDetails{} + } else { + obj = old.DeepCopy() + } + + obj.PotentiallyInconsistent = util.NewBool(backupMeta.PotentiallyInconsistent) + obj.SizeInBytes = backupMeta.SizeInBytes + obj.CreationTimestamp = v1.Time{ + Time: backupMeta.DateTime, + } + obj.NumberOfDBServers = backupMeta.NumberOfDBServers + obj.Version = backupMeta.Version + obj.ID = string(backupMeta.ID) + + return obj +} \ No newline at end of file diff --git a/pkg/backup/handlers/arango/backup/util.go b/pkg/backup/handlers/arango/backup/util.go index 107de594d..f09ebd4b1 100644 --- a/pkg/backup/handlers/arango/backup/util.go +++ b/pkg/backup/handlers/arango/backup/util.go @@ -23,14 +23,8 @@ package backup import ( - "fmt" - "reflect" "strings" - "github.com/arangodb/go-driver" - "github.com/arangodb/kube-arangodb/pkg/backup/utils" - "github.com/rs/zerolog/log" - clientBackup "github.com/arangodb/kube-arangodb/pkg/generated/clientset/versioned/typed/backup/v1alpha" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -50,49 +44,6 @@ var ( } ) -func switchTemporaryError(err error, status backupApi.ArangoBackupStatus) (backupApi.ArangoBackupStatus, error) { - if checkTemporaryError(err) { - return backupApi.ArangoBackupStatus{}, err - } - - return createFailedState(err, status), nil -} - -func createFailMessage(state state.State, message string) string { - return fmt.Sprintf("Failed State %s: %s", state, message) -} - -func createFailedState(err error, status backupApi.ArangoBackupStatus) backupApi.ArangoBackupStatus { - e := log.Error().Err(err).Str("type", reflect.TypeOf(err).String()) - if c, ok := err.(utils.Causer); ok { - e = e.AnErr("caused", c.Cause()).Str("causedType", reflect.TypeOf(c.Cause()).String()).Str("causedError", fmt.Sprintf("%v", c.Cause())) - - if a, ok := c.Cause().(driver.ArangoError); ok { - e = e.Str("aMsg", a.ErrorMessage).Int("aCode", a.Code).Int("aNum", a.ErrorNum).Str("aMsg", a.ErrorMessage).Bool("aTemp", a.Temporary()) - } - } - e.Msgf("Error %v", err) - - newStatus := status.DeepCopy() - - newStatus.ArangoBackupState = newState(backupApi.ArangoBackupStateFailed, createFailMessage(status.State, err.Error()), nil) - - newStatus.Available = false - - return *newStatus -} - -func newState(state state.State, message string, progress *backupApi.ArangoBackupProgress) backupApi.ArangoBackupState { - return backupApi.ArangoBackupState{ - State: state, - Time: meta.Now(), - - Message: message, - - Progress: progress, - } -} - func inProgress(backup *backupApi.ArangoBackup) bool { for _, state := range progressStates { if state == backup.Status.State { @@ -107,7 +58,7 @@ func isBackupRunning(backup *backupApi.ArangoBackup, client clientBackup.ArangoB backups, err := client.List(meta.ListOptions{}) if err != nil { - return false, err + return false, newTemporaryError(err) } for _, existingBackup := range backups.Items { diff --git a/tests/backup_test.go b/tests/backup_test.go index 46bdf8383..312ef56db 100644 --- a/tests/backup_test.go +++ b/tests/backup_test.go @@ -25,6 +25,7 @@ package tests import ( "context" "fmt" + "github.com/arangodb/kube-arangodb/pkg/backup/utils" "os" "strings" "testing" @@ -199,9 +200,18 @@ func newBackupPolicy(name, schedule string, labels map[string]string, options *E } func skipIfBackupUnavailable(t *testing.T, client driver.Client) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - if _, err := client.Backup().List(ctx, nil); err != nil { + err := utils.Retry(10, time.Second, func() error { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + if _, err := client.Backup().List(ctx, nil); err != nil { + t.Logf("Backup API not yet ready: %s", err.Error()) + return err + } + + return nil + }) + + if err != nil { t.Skipf("Backup API not available: %s", err.Error()) } } @@ -295,6 +305,15 @@ func timeoutWaitForBackups(t *testing.T, backupClient backupClient.ArangoBackupI } } +func compareBackup(t *testing.T, meta driver.BackupMeta, backup *backupApi.ArangoBackup) { + require.NotNil(t, backup.Status.Backup) + require.Equal(t, meta.Version, backup.Status.Backup.Version) + require.True(t, meta.SizeInBytes > 0) + require.True(t, meta.NumberOfDBServers == 2) + require.True(t, meta.SizeInBytes == backup.Status.Backup.SizeInBytes) + require.True(t, meta.NumberOfDBServers == backup.Status.Backup.NumberOfDBServers) +} + func TestBackupCluster(t *testing.T) { longOrSkip(t) c := client.MustNewInCluster() @@ -401,7 +420,7 @@ func TestBackupCluster(t *testing.T) { found, meta, err := statBackupMeta(databaseClient, driver.BackupID(backupID)) require.NoError(t, err, "Backup test failed: %s", err) require.True(t, found) - require.Equal(t, meta.Version, backup.Status.Backup.Version) + compareBackup(t, meta, backup) }) t.Run("create-upload backup", func(t *testing.T) { @@ -418,7 +437,7 @@ func TestBackupCluster(t *testing.T) { found, meta, err := statBackupMeta(databaseClient, driver.BackupID(backupID)) require.NoError(t, err, "Backup test failed: %s", err) require.True(t, found) - require.Equal(t, meta.Version, backup.Status.Backup.Version) + compareBackup(t, meta, backup) require.Nil(t, backup.Status.Backup.Uploaded) require.Nil(t, backup.Status.Backup.Downloaded) @@ -439,7 +458,7 @@ func TestBackupCluster(t *testing.T) { found, meta, err = statBackupMeta(databaseClient, driver.BackupID(backupID)) require.NoError(t, err, "Backup test failed: %s", err) require.True(t, found) - require.Equal(t, meta.Version, backup.Status.Backup.Version) + compareBackup(t, meta, backup) require.NotNil(t, backup.Status.Backup.Uploaded, "Upload flag is nil") require.Nil(t, backup.Status.Backup.Downloaded) }) @@ -452,7 +471,7 @@ func TestBackupCluster(t *testing.T) { found, meta, err := statBackupMeta(databaseClient, id) require.NoError(t, err, "Backup test failed: %s", err) require.True(t, found) - require.Equal(t, meta.Version, backup.Status.Backup.Version) + compareBackup(t, meta, backup) // now remove the backup backupClient.Delete(name, &metav1.DeleteOptions{}) @@ -482,7 +501,7 @@ func TestBackupCluster(t *testing.T) { // create a local backup manually id, _, err := databaseClient.Backup().Create(nil, nil) require.NoError(t, err, "Creating backup failed: %s", err) - found, _, err := statBackupMeta(databaseClient, driver.BackupID(id)) + found, meta, err := statBackupMeta(databaseClient, driver.BackupID(id)) require.NoError(t, err, "Backup test failed: %s", err) require.True(t, found) @@ -513,7 +532,7 @@ func TestBackupCluster(t *testing.T) { backup, err = waitUntilBackup(deploymentClient, backup.GetName(), ns, backupIsAvailable) require.NoError(t, err, "backup did not become available: %s", err) require.Equal(t, backupApi.ArangoBackupStateReady, backup.Status.State) - require.NotNil(t, backup.Status.Backup) + compareBackup(t, meta, backup) require.NotNil(t, backup.Status.Backup.Imported) require.True(t, *backup.Status.Backup.Imported) })