Arango Backup support in Operator

.gitattributes

@@ -0,0 +1,2 @@
+pkg/generated/** linguist-generated
+**/zz_generated.deepcopy.go linguist-generated

Makefile

@@ -71,6 +71,7 @@ endif
 MANIFESTPATHCRD := manifests/arango-crd$(MANIFESTSUFFIX).yaml
 MANIFESTPATHDEPLOYMENT := manifests/arango-deployment$(MANIFESTSUFFIX).yaml
 MANIFESTPATHDEPLOYMENTREPLICATION := manifests/arango-deployment-replication$(MANIFESTSUFFIX).yaml
+MANIFESTPATHBACKUP := manifests/arango-backup$(MANIFESTSUFFIX).yaml
 MANIFESTPATHSTORAGE := manifests/arango-storage$(MANIFESTSUFFIX).yaml
 MANIFESTPATHTEST := manifests/arango-test$(MANIFESTSUFFIX).yaml
 ifndef DEPLOYMENTNAMESPACE
@@ -111,7 +112,7 @@ TESTLENGTHOPTIONS := -test.short
 TESTTIMEOUT := 30m
 ifeq ($(LONG), 1)
 	TESTLENGTHOPTIONS :=
-	TESTTIMEOUT := 240m
+	TESTTIMEOUT := 300m
 endif
 ifdef VERBOSE
 	TESTVERBOSEOPTIONS := -v
@@ -151,6 +152,14 @@ allall: all
 # Tip: Run `eval $(minikube docker-env)` before calling make if you're developing on minikube.
 #
 
+.PHONY: fmt
+fmt:
+	golangci-lint run --no-config --issues-exit-code=1 --deadline=30m --disable-all --enable=deadcode --enable=gocyclo \
+	                  --enable=golint --enable=varcheck --enable=structcheck --enable=maligned --enable=errcheck \
+	                  --enable=dupl --enable=ineffassign --enable=interfacer --enable=unconvert --enable=goconst \
+	                  --enable=gosec --enable=megacheck --exclude-use-default=false \
+	                  $(ROOTDIR)/pkg/backup/...
+
 .PHONY: build
 build: docker manifests
 
@@ -182,7 +191,7 @@ update-generated:
 		"all" \
 		"github.com/arangodb/kube-arangodb/pkg/generated" \
 		"github.com/arangodb/kube-arangodb/pkg/apis" \
-		"deployment:v1alpha replication:v1alpha storage:v1alpha" \
+		"deployment:v1alpha replication:v1alpha storage:v1alpha backup:v1alpha" \
 		--go-header-file "./tools/codegen/boilerplate.go.txt" \
 		$(VERIFYARGS)
 
@@ -236,7 +245,8 @@ manifests-operator-deployment: helm
 	@$(HELM_CMD) \
 	     --set "operator.features.deployment=true" \
 	     --set "operator.features.deploymentReplications=false" \
-	     --set "operator.features.storage=false" > "$(MANIFESTPATHDEPLOYMENT)"
+	     --set "operator.features.storage=false" \
+	     --set "operator.features.backup=false" > "$(MANIFESTPATHDEPLOYMENT)"
 
 .PHONY: manifests-operator-deployment-replication
 manifests-operator-deployment-replication: export CHART_NAME := kube-arangodb
@@ -246,7 +256,8 @@ manifests-operator-deployment-replication: helm
 	@$(HELM_CMD) \
 	     --set "operator.features.deployment=false" \
 	     --set "operator.features.deploymentReplications=true" \
-	     --set "operator.features.storage=false" > "$(MANIFESTPATHDEPLOYMENTREPLICATION)"
+	     --set "operator.features.storage=false" \
+	     --set "operator.features.backup=false" > "$(MANIFESTPATHDEPLOYMENTREPLICATION)"
 
 .PHONY: manifests-operator-storage
 manifests-operator-storage: export CHART_NAME := kube-arangodb
@@ -256,10 +267,22 @@ manifests-operator-storage: helm
 	@$(HELM_CMD) \
 	     --set "operator.features.deployment=false" \
 	     --set "operator.features.deploymentReplications=false" \
-	     --set "operator.features.storage=true" > "$(MANIFESTPATHSTORAGE)"
+	     --set "operator.features.storage=true" \
+	     --set "operator.features.backup=false" > "$(MANIFESTPATHSTORAGE)"
+
+.PHONY: manifests-operator-backup
+manifests-operator-backup: export CHART_NAME := kube-arangodb
+manifests-operator-backup: export NAME := backup
+manifests-operator-backup: helm
+	@echo Building manifests for Operator Backup - $(MANIFESTPATHBACKUP)
+	@$(HELM_CMD) \
+	     --set "operator.features.deployment=false" \
+	     --set "operator.features.deploymentReplications=false" \
+	     --set "operator.features.storage=false" \
+	     --set "operator.features.backup=true" > "$(MANIFESTPATHBACKUP)"
 
 .PHONY: manifests-operator
-manifests-operator: manifests-operator-deployment manifests-operator-deployment-replication manifests-operator-storage
+manifests-operator: manifests-operator-deployment manifests-operator-deployment-replication manifests-operator-storage manifests-operator-backup
 
 .PHONY: chart-crd
 chart-crd: export CHART_NAME := kube-arangodb-crd
@@ -281,6 +304,7 @@ manifests: helm manifests-crd manifests-operator manifests-test chart-crd chart-
 .PHONY: run-unit-tests
 run-unit-tests: $(SOURCES)
 	go test $(TESTVERBOSEOPTIONS) \
+		$(REPOPATH)/pkg/apis/backup/v1alpha \
 		$(REPOPATH)/pkg/apis/deployment/v1alpha \
 		$(REPOPATH)/pkg/apis/replication/v1alpha \
 		$(REPOPATH)/pkg/apis/storage/v1alpha \
@@ -290,7 +314,8 @@ run-unit-tests: $(SOURCES)
 		$(REPOPATH)/pkg/util/k8sutil \
 		$(REPOPATH)/pkg/util/k8sutil/test \
 		$(REPOPATH)/pkg/util/probe \
-		$(REPOPATH)/pkg/util/validation
+		$(REPOPATH)/pkg/util/validation \
+		$(REPOPATH)/pkg/backup/...
 
 $(TESTBIN): $(GOBUILDDIR) $(SOURCES)
 	@mkdir -p $(BINDIR)
@@ -318,9 +343,11 @@ endif
 	kubectl apply -f $(MANIFESTPATHSTORAGE)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENT)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION)
+	kubectl apply -f $(MANIFESTPATHBACKUP)
 	kubectl apply -f $(MANIFESTPATHTEST)
 	$(ROOTDIR)/scripts/kube_create_storage.sh $(DEPLOYMENTNAMESPACE)
 	$(ROOTDIR)/scripts/kube_create_license_key_secret.sh "$(DEPLOYMENTNAMESPACE)" '$(ENTERPRISELICENSE)'
+	$(ROOTDIR)/scripts/kube_create_backup_remote_secret.sh "$(DEPLOYMENTNAMESPACE)" '$(TEST_REMOTE_SECRET)'
 
 .PHONY: run-tests
 run-tests: docker-test
@@ -336,10 +363,12 @@ endif
 	kubectl apply -f $(MANIFESTPATHSTORAGE)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENT)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION)
+	kubectl apply -f $(MANIFESTPATHBACKUP)
 	kubectl apply -f $(MANIFESTPATHTEST)
 	$(ROOTDIR)/scripts/kube_create_storage.sh $(DEPLOYMENTNAMESPACE)
 	$(ROOTDIR)/scripts/kube_create_license_key_secret.sh "$(DEPLOYMENTNAMESPACE)" '$(ENTERPRISELICENSE)'
-	$(ROOTDIR)/scripts/kube_run_tests.sh $(DEPLOYMENTNAMESPACE) $(TESTIMAGE) "$(ARANGODIMAGE)" '$(ENTERPRISEIMAGE)' '$(TESTTIMEOUT)' '$(TESTLENGTHOPTIONS)' '$(TESTOPTIONS)'
+	$(ROOTDIR)/scripts/kube_create_backup_remote_secret.sh "$(DEPLOYMENTNAMESPACE)" '$(TEST_REMOTE_SECRET)'
+	$(ROOTDIR)/scripts/kube_run_tests.sh $(DEPLOYMENTNAMESPACE) $(TESTIMAGE) "$(ARANGODIMAGE)" '$(ENTERPRISEIMAGE)' '$(TESTTIMEOUT)' '$(TESTLENGTHOPTIONS)' '$(TESTOPTIONS)' '$(TEST_REMOTE_REPOSITORY)'
 
 $(DURATIONTESTBIN): $(SOURCES)
 	CGO_ENABLED=0 go build -installsuffix cgo -ldflags "-X main.projectVersion=$(VERSION) -X main.projectBuild=$(COMMIT)" -o $(DURATIONTESTBINNAME) $(REPOPATH)/tests/duration
@@ -430,6 +459,7 @@ delete-operator:
 	kubectl delete -f $(MANIFESTPATHTEST) --ignore-not-found
 	kubectl delete -f $(MANIFESTPATHDEPLOYMENT) --ignore-not-found
 	kubectl delete -f $(MANIFESTPATHDEPLOYMENTREPLICATION) --ignore-not-found
+	kubectl delete -f $(MANIFESTPATHBACKUP) --ignore-not-found
 	kubectl delete -f $(MANIFESTPATHSTORAGE) --ignore-not-found
 	kubectl delete -f $(MANIFESTPATHCRD) --ignore-not-found
 
@@ -439,6 +469,7 @@ redeploy-operator: delete-operator manifests
 	kubectl apply -f $(MANIFESTPATHSTORAGE)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENT)
 	kubectl apply -f $(MANIFESTPATHDEPLOYMENTREPLICATION)
+	kubectl apply -f $(MANIFESTPATHBACKUP)
 	kubectl apply -f $(MANIFESTPATHTEST)
 	kubectl get pods
 

chart/kube-arangodb-crd/templates/backup-policy.yaml

@@ -0,0 +1,49 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+    name: arangobackups.backup.arangodb.com
+    labels:
+        app.kubernetes.io/name: {{ template "kube-arangodb-crd.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        release: {{ .Release.Name }}
+spec:
+    group: backup.arangodb.com
+    additionalPrinterColumns:
+        - JSONPath: .spec.policyName
+          description: Policy name
+          name: Policy
+          type: string
+        - JSONPath: .spec.deployment.name
+          description: Deployment name
+          name: Deployment
+          type: string
+        - JSONPath: .status.backup.version
+          description: Backup Version
+          name: Version
+          type: string
+        - JSONPath: .status.backup.createdAt
+          description: Backup Creation Timestamp
+          name: Created
+          type: string
+        - JSONPath: .status.state
+          description: The actual state of the ArangoBackup
+          name: State
+          type: string
+        - JSONPath: .status.message
+          priority: 1
+          description: Message of the ArangoBackup object
+          name: Message
+          type: string
+    names:
+        kind: ArangoBackup
+        listKind: ArangoBackupList
+        plural: arangobackups
+        shortNames:
+            - arangobackup
+        singular: arangobackup
+    scope: Namespaced
+    version: v1alpha
+    subresources:
+        status: {}

chart/kube-arangodb-crd/templates/backup.yaml

@@ -0,0 +1,38 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+    name: arangobackuppolicies.backup.arangodb.com
+    labels:
+        app.kubernetes.io/name: {{ template "kube-arangodb-crd.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        release: {{ .Release.Name }}
+spec:
+    group: backup.arangodb.com
+    additionalPrinterColumns:
+        - JSONPath: .spec.schedule
+          description: Schedule
+          name: Schedule
+          type: string
+        - JSONPath: .status.scheduled
+          description: Scheduled
+          name: Scheduled
+          type: string
+        - JSONPath: .status.message
+          priority: 1
+          description: Message of the ArangoBackupPolicy object
+          name: Message
+          type: string
+    names:
+        kind: ArangoBackupPolicy
+        listKind: ArangoBackupPolicyList
+        plural: arangobackuppolicies
+        shortNames:
+            - arangobackuppolicy
+            - arangobp
+        singular: arangobackuppolicy
+    scope: Namespaced
+    version: v1alpha
+    subresources:
+        status: {}

chart/kube-arangodb/README.md

@@ -133,6 +133,12 @@ Define if ArangoLocalStorage Operator should be enabled.
 
 Default: `false`
 
+### `operator.features.backup`
+
+Define if ArangoBackup Operator should be enabled.
+
+Default: `false`
+
 ### `rbac.enabled`
 
 Define if RBAC should be enabled.

chart/kube-arangodb/templates/backup-operator/cluster-role-binding.yaml

@@ -0,0 +1,26 @@
+{{ if .Values.rbac.enabled -}}
+{{ if .Values.operator.features.backup -}}
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+    name: {{ template "kube-arangodb.rbac" . }}-storage
+    namespace: {{ .Release.Namespace }}
+    labels:
+        app.kubernetes.io/name: {{ template "kube-arangodb.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        release: {{ .Release.Name }}
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: ClusterRole
+    name: {{ template "kube-arangodb.rbac" . }}-backup
+subjects:
+    - kind: ServiceAccount
+      name: {{ template "kube-arangodb.operatorName" . }}
+      namespace: {{ .Release.Namespace }}
+
+
+{{- end }}
+{{- end }}

chart/kube-arangodb/templates/backup-operator/cluster-role.yaml

@@ -0,0 +1,20 @@
+{{ if .Values.rbac.enabled -}}
+{{ if .Values.operator.features.backup -}}
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+    name: {{ template "kube-arangodb.rbac" . }}-backup
+    namespace: {{ .Release.Namespace }}
+    labels:
+        app.kubernetes.io/name: {{ template "kube-arangodb.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        release: {{ .Release.Name }}
+rules:
+    - apiGroups: ["apiextensions.k8s.io"]
+      resources: ["customresourcedefinitions"]
+      verbs: ["get", "list", "watch"]
+{{- end }}
+{{- end }}

chart/kube-arangodb/templates/backup-operator/role-binding.yaml

@@ -0,0 +1,26 @@
+{{ if .Values.rbac.enabled -}}
+{{ if .Values.operator.features.backup -}}
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+    name: {{ template "kube-arangodb.rbac" . }}-backup
+    namespace: {{ .Release.Namespace }}
+    labels:
+        app.kubernetes.io/name: {{ template "kube-arangodb.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        release: {{ .Release.Name }}
+roleRef:
+    apiGroup: rbac.authorization.k8s.io
+    kind: Role
+    name: {{ template "kube-arangodb.rbac" . }}-backup
+subjects:
+    - kind: ServiceAccount
+      name: {{ template "kube-arangodb.operatorName" . }}
+      namespace: {{ .Release.Namespace }}
+
+
+{{- end }}
+{{- end }}

chart/kube-arangodb/templates/backup-operator/role.yaml

@@ -0,0 +1,32 @@
+{{ if .Values.rbac.enabled -}}
+{{ if .Values.operator.features.backup -}}
+
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+    name: {{ template "kube-arangodb.rbac" . }}-backup
+    namespace: {{ .Release.Namespace }}
+    labels:
+        app.kubernetes.io/name: {{ template "kube-arangodb.name" . }}
+        helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
+        app.kubernetes.io/managed-by: {{ .Release.Service }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        release: {{ .Release.Name }}
+rules:
+    - apiGroups: [""]
+      resources: ["pods", "services", "endpoints"]
+      verbs: ["get", "update"]
+    - apiGroups: [""]
+      resources: ["secrets"]
+      verbs: ["get"]
+    - apiGroups: ["apps"]
+      resources: ["deployments", "replicasets"]
+      verbs: ["get"]
+    - apiGroups: ["backup.arangodb.com"]
+      resources: ["arangobackuppolicies", "arangobackuppolicies/status", "arangobackups", "arangobackups/status"]
+      verbs: ["*"]
+    - apiGroups: ["database.arangodb.com"]
+      resources: ["arangodeployments"]
+      verbs: ["get", "list", "watch"]
+{{- end }}
+{{- end }}

chart/kube-arangodb/templates/deployment-operator/role.yaml

@@ -25,6 +25,9 @@ rules:
     - apiGroups: ["policy"]
       resources: ["poddisruptionbudgets"]
       verbs: ["get", "create", "delete"]
+    - apiGroups: ["backup.arangodb.com"]
+      resources: ["arangobackuppolicies", "arangobackups"]
+      verbs: ["get", "list", "watch"]
     - apiGroups: ["monitoring.coreos.com"]
       resources: ["servicemonitors"]
       verbs: ["get", "create", "delete"]

chart/kube-arangodb/templates/deployment.yaml

@@ -59,6 +59,9 @@ spec:
 {{- end -}}
 {{ if .Values.operator.features.storage }}
                     - --operator.storage
+{{- end }}
+{{ if .Values.operator.features.backup }}
+                    - --operator.backup
 {{- end }}
                     - --chaos.allowed={{ .Values.operator.allowChaos }}
                   env:

chart/kube-arangodb/values.yaml

@@ -29,6 +29,7 @@ operator:
     deployment: true
     deploymentReplications: true
     storage: false
+    backup: false
 
 rbac:
   enabled: true