Skip to content

Commit

Permalink
Add BackupBatch support (#985)
Browse files Browse the repository at this point in the history 
* Add BackupBatch support

* Rebase on master + Refactor

* Fix license header

Signed-off-by: Tamal Saha <tamal@appscode.com>

* Fix issues and Add BackupBatch e2e test

* Run `make verify-module`

* Fix `handlePreTaskHookFailure` func

* backupConfigurationTemplates -> members

Signed-off-by: Tamal Saha <tamal@appscode.com>

Co-authored-by: Md. Emruz Hossain <hossainemruz@gmail.com>
Co-authored-by: Tamal Saha <tamal@appscode.com>
  • Loading branch information
@suaas21 @hossainemruz @tamalsaha
3 people committed Dec 31, 2019
1 parent 190e4cc commit 62ce832
Show file tree
Hide file tree
Showing 58 changed files with 16,318 additions and 3,865 deletions.
5 changes: 3 additions & 2 deletions api/api-rules/violation_exceptions.list
View file
Expand Up @@ -187,11 +187,11 @@ API rule violation: list_type_missing,kmodules.xyz/prober/api/v1,FormEntry,Value
API rule violation: list_type_missing,kmodules.xyz/prober/api/v1,HTTPPostAction,Form
API rule violation: list_type_missing,kmodules.xyz/prober/api/v1,HTTPPostAction,HTTPHeaders
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupBatchList,Items
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupBatchSpec,BackupConfigurationTemplates
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupBatchSpec,Members
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupBlueprintList,Items
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupConfigurationList,Items
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupSessionList,Items
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupSessionStatus,Stats
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupSessionStatus,Targets
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupTarget,Paths
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,BackupTarget,VolumeMounts
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,FunctionList,Items
Expand All @@ -210,6 +210,7 @@ API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,Rule,Paths
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,Rule,Snapshots
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,Rule,TargetHosts
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,Target,Stats
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,TaskList,Items
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,TaskRef,Params
API rule violation: list_type_missing,stash.appscode.dev/stash/apis/stash/v1beta1,TaskSpec,Steps
Expand Down
8 changes: 4 additions & 4 deletions api/crds/bindata.go
View file

Large diffs are not rendered by default.

6,578 changes: 3,988 additions & 2,590 deletions api/crds/stash.appscode.com_backupbatches.yaml
View file

Large diffs are not rendered by default.

154 changes: 88 additions & 66 deletions api/crds/stash.appscode.com_backupsessions.yaml
View file
Expand Up @@ -78,87 +78,109 @@ spec:
type: string
sessionDuration:
description: SessionDuration specify total time taken to complete current
backup session (sum of backup duration of all hosts)
backup session (sum of backup duration of all targets)
type: string
stats:
description: Stats shows statistics of individual hosts for this backup
session
targets:
description: Targets specify the backup status of individual targets
optional
items:
properties:
duration:
description: Duration indicates total time taken to complete backup
for this hosts
type: string
error:
description: Error indicates string value of error in case of
backup failure
type: string
hostname:
description: Hostname indicate name of the host that has been
backed up
type: string
phase:
description: Phase indicates backup phase of this host
description: Phase indicates backup phase of this target
type: string
snapshots:
description: Snapshots specifies the stats of individual snapshots
that has been taken for this host in current backup session
ref:
description: Ref refers to the backup target
properties:
apiVersion:
type: string
kind:
type: string
name:
type: string
type: object
stats:
description: Stats shows statistics of individual hosts for this
backup session
items:
properties:
fileStats:
description: FileStats shows statistics of files of this
snapshot
properties:
modifiedFiles:
description: ModifiedFiles shows total number of files
that has been modified since last backup
format: int64
type: integer
newFiles:
description: NewFiles shows total number of new files
that has been created since last backup
format: int64
type: integer
totalFiles:
description: TotalFiles shows total number of files
that has been backed up
format: int64
type: integer
unmodifiedFiles:
description: UnmodifiedFiles shows total number of files
that has not been changed since last backup
format: int64
type: integer
type: object
name:
description: Name indicates the name of the backup snapshot
created for this host
type: string
path:
description: Path indicates the directory that has been
backed up in this snapshot
duration:
description: Duration indicates total time taken to complete
backup for this hosts
type: string
processingTime:
description: ProcessingTime indicates time taken to process
the target data
error:
description: Error indicates string value of error in case
of backup failure
type: string
totalSize:
description: TotalSize indicates the size of data to backup
in target directory
hostname:
description: Hostname indicate name of the host that has
been backed up
type: string
uploaded:
description: Uploaded indicates size of data uploaded to
backend for this snapshot
phase:
description: Phase indicates backup phase of this host
type: string
snapshots:
description: Snapshots specifies the stats of individual
snapshots that has been taken for this host in current
backup session
items:
properties:
fileStats:
description: FileStats shows statistics of files of
this snapshot
properties:
modifiedFiles:
description: ModifiedFiles shows total number
of files that has been modified since last backup
format: int64
type: integer
newFiles:
description: NewFiles shows total number of new
files that has been created since last backup
format: int64
type: integer
totalFiles:
description: TotalFiles shows total number of
files that has been backed up
format: int64
type: integer
unmodifiedFiles:
description: UnmodifiedFiles shows total number
of files that has not been changed since last
backup
format: int64
type: integer
type: object
name:
description: Name indicates the name of the backup
snapshot created for this host
type: string
path:
description: Path indicates the directory that has
been backed up in this snapshot
type: string
processingTime:
description: ProcessingTime indicates time taken to
process the target data
type: string
totalSize:
description: TotalSize indicates the size of data
to backup in target directory
type: string
uploaded:
description: Uploaded indicates size of data uploaded
to backend for this snapshot
type: string
type: object
type: array
type: object
type: array
totalHosts:
description: TotalHosts specifies total number of hosts for this
target that will be backed up for a BackupSession
format: int32
type: integer
type: object
type: array
totalHosts:
description: TotalHosts specifies total number of hosts that will be
backed up for this BackupSession
format: int32
type: integer
type: object
type: object
version: v1beta1
Expand Down
75 changes: 40 additions & 35 deletions api/openapi-spec/swagger.json
View file
Expand Up @@ -10128,13 +10128,6 @@
"retentionPolicy"
],
"properties": {
"backupConfigurationTemplates": {
"description": "backupConfigurationTemplates is a list of backup configurations that are part of this batch",
"type": "array",
"items": {
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.BackupConfigurationTemplate"
}
},
"backupHistoryLimit": {
"description": "BackupHistoryLimit specifies the number of BackupSession and it's associate resources to keep. This is helpful for debugging purpose. Default: 1",
"type": "integer",
Expand All @@ -10148,6 +10141,13 @@
"description": "Actions that Stash should take in response to backup sessions. Cannot be updated.",
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.BackupHooks"
},
"members": {
"description": "members is a list of backup configurations that are part of this batch",
"type": "array",
"items": {
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.BackupConfigurationTemplateSpec"
}
},
"paused": {
"description": "Indicates that the BackupConfiguration is paused from taking backup. Default value is 'false'",
"type": "boolean"
Expand All @@ -10160,6 +10160,10 @@
"description": "RetentionPolicy indicates the policy to follow to clean old backup snapshots",
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1alpha1.RetentionPolicy"
},
"runtimeSettings": {
"description": "RuntimeSettings allow to specify Resources, NodeSelector, Affinity, Toleration, ReadinessProbe etc, and used to create service account for CronJob.",
"$ref": "#/definitions/xyz.kmodules.offshoot-api.api.v1.RuntimeSettings"
},
"schedule": {
"description": "Schedule specifies the schedule for invoking backup sessions",
"type": "string"
Expand Down Expand Up @@ -10400,25 +10404,6 @@
}
}
},
"dev.appscode.stash.stash.apis.stash.v1beta1.BackupConfigurationTemplate": {
"type": "object",
"properties": {
"apiVersion": {
"description": "APIVersion defines the versioned schema of this representation of an object. Servers should convert recognized schemas to the latest internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources",
"type": "string"
},
"kind": {
"description": "Kind is a string value representing the REST resource this object represents. Servers may infer this from the endpoint the client submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds",
"type": "string"
},
"metadata": {
"$ref": "#/definitions/xyz.kmodules.offshoot-api.api.v1.PartialObjectMeta"
},
"spec": {
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.BackupConfigurationTemplateSpec"
}
}
},
"dev.appscode.stash.stash.apis.stash.v1beta1.BackupConfigurationTemplateSpec": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -10559,20 +10544,15 @@
"type": "string"
},
"sessionDuration": {
"description": "SessionDuration specify total time taken to complete current backup session (sum of backup duration of all hosts)",
"description": "SessionDuration specify total time taken to complete current backup session (sum of backup duration of all targets)",
"type": "string"
},
"stats": {
"description": "Stats shows statistics of individual hosts for this backup session",
"targets": {
"description": "Targets specify the backup status of individual targets optional",
"type": "array",
"items": {
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.HostBackupStats"
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.Target"
}
},
"totalHosts": {
"description": "TotalHosts specifies total number of hosts that will be backed up for this BackupSession",
"type": "integer",
"format": "int32"
}
}
},
Expand Down Expand Up @@ -11074,6 +11054,31 @@
}
}
},
"dev.appscode.stash.stash.apis.stash.v1beta1.Target": {
"type": "object",
"properties": {
"phase": {
"description": "Phase indicates backup phase of this target",
"type": "string"
},
"ref": {
"description": "Ref refers to the backup target",
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.TargetRef"
},
"stats": {
"description": "Stats shows statistics of individual hosts for this backup session",
"type": "array",
"items": {
"$ref": "#/definitions/dev.appscode.stash.stash.apis.stash.v1beta1.HostBackupStats"
}
},
"totalHosts": {
"description": "TotalHosts specifies total number of hosts for this target that will be backed up for a BackupSession",
"type": "integer",
"format": "int32"
}
}
},
"dev.appscode.stash.stash.apis.stash.v1beta1.TargetRef": {
"type": "object",
"properties": {
Expand Down
53 changes: 53 additions & 0 deletions apis/constants.go
View file
Expand Up @@ -16,6 +16,8 @@ limitations under the License.

package apis

import "time"

const (
StashDockerRegistry = "STASH_DOCKER_REGISTRY"
StashDockerImage = "STASH_DOCKER_IMAGE"
Expand Down Expand Up @@ -92,3 +94,54 @@ const (
PreTaskHook = "pre-task-hook"
PostTaskHook = "post-task-hook"
)

const (
StashContainer = "stash"
StashInitContainer = "stash-init"
LocalVolumeName = "stash-local"
ScratchDirVolumeName = "stash-scratchdir"
TmpDirVolumeName = "tmp-dir"
TmpDirMountPath = "/tmp"
PodinfoVolumeName = "stash-podinfo"

RecoveryJobPrefix = "stash-recovery-"
ScaledownCronPrefix = "stash-scaledown-cron-"
CheckJobPrefix = "stash-check-"

AnnotationRestic = "restic"
AnnotationRecovery = "recovery"
AnnotationOperation = "operation"
AnnotationOldReplica = "old-replica"

OperationRecovery = "recovery"
OperationCheck = "check"

AppLabelStash = "stash"
AppLabelStashV1Beta1 = "stash-v1beta1"
OperationScaleDown = "scale-down"

RepositoryFinalizer = "stash"
SnapshotIDLength = 8

ModelSidecar = "sidecar"
ModelCronJob = "cronjob"
LabelApp = "app"
LabelInvokerType = StashKey + "/invoker-type"
LabelInvokerName = StashKey + "/invoker-name"
StashSecretVolume = "stash-secret-volume"
StashSecretMountDir = "/etc/stash/repository/secret"

KeyPodName = "POD_NAME"
KeyNodeName = "NODE_NAME"
KeyPodOrdinal = "POD_ORDINAL"

RetryInterval = 50 * time.Millisecond
ReadinessTimeout = 2 * time.Minute
)

const (
CallerWebhook = "webhook"
CallerController = "controller"
PushgatewayLocalURL = "http://localhost:56789"
DefaultHost = "host-0"
)

0 comments on commit 62ce832

Please sign in to comment.