Phase1: PVC snapshot backup/restore/delete (overview of functions) (#…

…3981)

* PVC snapshot backup/restore/delete

* Address review comments

* Modified BP and kanister func

* address review changes

* address review changes

* address review comments

* improve unit test

* update bblueprint with render Artifacts changes

* address review comments

examples/picture-gallery-pvc-snapshot/README.md

@@ -0,0 +1,3 @@
+# Walkthrough of Picture Gallery
+
+The blueprint for this application is in progress. Once it is ready, we will be able to take PVC screenshots, restore PVCs and delete the backup data.

examples/picture-gallery-pvc-snapshot/backup-actionset.yaml

@@ -0,0 +1,17 @@
+apiVersion: cr.kanister.io/v1alpha1
+kind: ActionSet
+metadata:
+  generateName: pic-gal-pvc-snapshot-
+  namespace: kanister
+spec:
+  actions:
+  - name: backup
+    blueprint: picture-gallery
+    object:
+      kind: Deployment
+      name: picture-gallery
+      namespace: default
+    profile:
+      kind: Profile
+      name: default-profile
+      namespace: kanister

examples/picture-gallery-pvc-snapshot/blueprint.yaml

@@ -0,0 +1,51 @@
+apiVersion: cr.kanister.io/v1alpha1
+kind: Blueprint
+metadata:
+  name: picture-gallery
+  namespace: kanister
+actions:
+  backup:
+    type: Deployment
+    outputArtifacts:
+      backupLocation:
+        keyValue:
+          path: "{{ .Phases.backupVolume.Output.backupLocation }}"
+    phases:
+    - func: CreateVolumeSnapshot
+      name: backupVolume
+      args:
+        namespace: "{{ .Deployment.Namespace }}"
+  restore:
+    type: Deployment
+    inputArtifactNames:
+      - backupLocation
+    phases:
+    - func: ScaleWorkload
+      name: shutdownPod
+      args:
+        namespace: "{{ .Deployment.Namespace }}"
+        name: "{{ .Deployment.Name }}"
+        kind: Deployment
+        replicas: 0
+    - func: CreateVolumeFromSnapshot
+      name: restoreVolume
+      args:
+        namespace: "{{ .Deployment.Namespace }}"
+        snapshots: "{{ .ArtifactsIn.backupLocation.KeyValue.path }}"
+    - func: ScaleWorkload
+      name: bringupPod
+      args:
+        namespace: "{{ .Deployment.Namespace }}"
+        name: "{{ .Deployment.Name }}"
+        kind: Deployment
+        replicas: 1
+  delete:
+    type: Deployment
+    inputArtifactNames:
+      - backupLocation
+    phases:
+    - func: DeleteVolumeSnapshot
+      name: deleteVolumeSnapshot
+      args:
+        namespace: "{{ .Deployment.Namespace }}"
+        snapshots: "{{ .ArtifactsIn.backupLocation.KeyValue.path }}"

examples/picture-gallery-pvc-snapshot/picture-gallery-deployment.yaml

@@ -0,0 +1,122 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: picture-gallery
+  labels:
+    app: picture-gallery
+spec:
+  ports:
+    - port: 80
+  selector:
+    app: picture-gallery
+    tier: frontend
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pic-gal-php-cfg
+  labels:
+    app: picture-gallery
+    contains: php-config
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pic-gal-pics
+  labels:
+    app: picture-gallery
+    contains: uploaded-pictures
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 2Gi
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: pic-gal-mysql
+  labels:
+    app: picture-gallery
+    contains: mysql-dbs
+spec:
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 1Gi
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: picture-gallery
+  labels:
+    app: picture-gallery
+spec:
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      app: picture-gallery
+  template:
+    metadata:
+      labels:
+        app: picture-gallery
+        tier: frontend
+    spec:
+      containers:
+      - image: kastenio/picturegallery:70307e
+        livenessProbe:
+          httpGet:
+            path: /
+            port: 80
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        name: picture-gallery
+        ports:
+        - containerPort: 80
+          name: picture-gallery
+        volumeMounts:
+        - name: pic-gal-php-cfg
+          mountPath: /data
+        - name: pic-gal-pic
+          mountPath: /uploads
+        - name: pic-gal-mysql
+          mountPath: /mysql   
+      volumes:
+      - name: pic-gal-php-cfg
+        persistentVolumeClaim:
+          claimName: pic-gal-php-cfg
+      - name: pic-gal-pic
+        persistentVolumeClaim:
+          claimName: pic-gal-pics
+      - name: pic-gal-mysql
+        persistentVolumeClaim:
+          claimName: pic-gal-mysql
+---
+kind: Ingress
+apiVersion: extensions/v1beta1
+#creating ingress for picture-gallery
+#is accessible by cluster.domainname/picture-gallery
+metadata:
+  name: picture-gallery
+  annotations:
+    kubernetes.io/ingress.class: "nginx"
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+    nginx.ingress.kubernetes.io/proxy-body-size: "100m"
+spec:
+  rules:
+  - http:
+      paths:
+      - path: /picture-gallery
+        backend:
+          serviceName: picture-gallery
+          servicePort: 80

pkg/function/create_volume_from_snapshot.go

@@ -0,0 +1,96 @@
+package function
+
+import (
+	"context"
+	"encoding/json"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/client-go/kubernetes"
+
+	kanister "github.com/kanisterio/kanister/pkg"
+	"github.com/kanisterio/kanister/pkg/kube"
+	"github.com/kanisterio/kanister/pkg/objectstore"
+	"github.com/kanisterio/kanister/pkg/param"
+)
+
+func init() {
+	kanister.Register(&createVolumeFromSnapshotFunc{})
+}
+
+var (
+	_ kanister.Func = (*createVolumeFromSnapshotFunc)(nil)
+)
+
+const (
+	CreateVolumeFromSnapshotNamespaceArg = "namespace"
+	CreateVolumeFromSnapshotPathArg      = "snapshots"
+)
+
+type createVolumeFromSnapshotFunc struct{}
+
+func (*createVolumeFromSnapshotFunc) Name() string {
+	return "CreateVolumeFromSnapshot"
+}
+
+func createVolumeFromSnapshot(ctx context.Context, cli kubernetes.Interface, namespace, snapshotPath string, profile *param.Profile) error {
+	data, err := objectstore.GetData(ctx, profile, objectstore.ProviderTypeS3, profile.Location.S3Compliant.Bucket, snapshotPath, "manifest.txt")
+	if err != nil {
+		return err
+	}
+	PVCData := []VolumeSnapshotInfo{}
+	err = json.Unmarshal(data, &PVCData)
+	if err != nil {
+		return errors.Wrapf(err, "Could not decode JSON data")
+	}
+	for _, pvcInfo := range PVCData {
+		var storageType string
+		switch pvcInfo.StorageType {
+		// TODO: use constants once blockstorage is moved to kanister repo
+		case "EBS":
+			storageType = "EBS"
+		case "GPD":
+			storageType = "GPD"
+		case "AD":
+			storageType = "AD"
+		case "Cinder":
+			storageType = "Cinder"
+		case "Ceph":
+			storageType = "Ceph"
+		default:
+			return errors.Errorf("Storage type %s not supported!", pvcInfo.StorageType)
+		}
+		log.Infof("snapshotId: %s, StorageType: %s, region: %s", pvcInfo.SnapshotID, storageType, pvcInfo.Region)
+		if err := createPVCFromSnapshot(); err != nil {
+			return errors.Wrapf(err, "Could not create PVC")
+		}
+	}
+	return nil
+}
+
+func createPVCFromSnapshot() error {
+	return errors.Wrapf(createPV(), "Could not create PV")
+}
+
+func createPV() error {
+	return nil
+}
+
+func (kef *createVolumeFromSnapshotFunc) Exec(ctx context.Context, tp param.TemplateParams, args map[string]interface{}) (map[string]interface{}, error) {
+	cli, err := kube.NewClient()
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed to create Kubernetes client")
+	}
+	var namespace, snapshots string
+	if err = Arg(args, CreateVolumeFromSnapshotNamespaceArg, &namespace); err != nil {
+		return nil, err
+	}
+	if err = Arg(args, CreateVolumeFromSnapshotPathArg, &snapshots); err != nil {
+		return nil, err
+	}
+	return nil, createVolumeFromSnapshot(ctx, cli, namespace, snapshots, tp.Profile)
+}
+
+func (*createVolumeFromSnapshotFunc) RequiredArgs() []string {
+	return []string{CreateVolumeFromSnapshotNamespaceArg, CreateVolumeFromSnapshotPathArg}
+}