Merge pull request #31251 from rootfs/rbd-prov3

Automatic merge from submit-queue support storage class in Ceph RBD volume replace WIP PR #30959, using PV annotation idea from @jsafrane @kubernetes/sig-storage @johscheuer @elsonrodriguez
kubernetes · Sep 10, 2016 · 34141a7 · 34141a7
2 parents a5d6bf4 + 0c3b2f4
commit 34141a7
Show file tree

Hide file tree

Showing 11 changed files with 511 additions and 21 deletions.
diff --git a/cmd/kube-controller-manager/app/plugins.go b/cmd/kube-controller-manager/app/plugins.go
@@ -45,6 +45,7 @@ import (
 	"k8s.io/kubernetes/pkg/volume/glusterfs"
 	"k8s.io/kubernetes/pkg/volume/host_path"
 	"k8s.io/kubernetes/pkg/volume/nfs"
+	"k8s.io/kubernetes/pkg/volume/rbd"
 	"k8s.io/kubernetes/pkg/volume/vsphere_volume"
 )
 
@@ -102,7 +103,8 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config componen
 	}
 	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...)
 	allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
-
+	// add rbd provisioner
+	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
 	if cloud != nil {
 		switch {
 		case aws.ProviderName == cloud.ProviderName():

diff --git a/examples/experimental/persistent-volume-provisioning/README.md b/examples/experimental/persistent-volume-provisioning/README.md
@@ -121,6 +121,32 @@ parameters:
 * `type`: [VolumeType](http://docs.openstack.org/admin-guide/dashboard-manage-volumes.html) created in Cinder. Default is empty.
 * `availability`: Availability Zone. Default is empty.
 
+#### Ceph RBD
+
+```yaml                                                                                                                                                                                                          
+  apiVersion: extensions/v1beta1                                                                                                                                                                                   
+  kind: StorageClass                                                                                                                                                                                               
+  metadata: 
+   name: fast
+provisioner: kubernetes.io/rbd
+parameters:
+    monitors: 10.16.153.105:6789
+    adminId: kube
+    adminSecretName: ceph-secret
+    adminSecretNamespace: kube-system
+    pool: kube
+    userId: kube
+    userSecretName: ceph-secret-user
+```
+
+* `monitors`: Ceph monitors, comma delimited. It is required.
+* `adminId`: Ceph client ID that is capable of creating images in the pool. Default is "admin".
+* `adminSecret`: Secret Name for `adminId`. It is required.
+* `adminSecretNamespace`: The namespace for `adminSecret`. Default is "default".
+* `pool`: Ceph RBD pool. Default is "rbd".
+* `userId`: Ceph client ID that is used to map the RBD image. Default is the same as `adminId`.
+* `userSecretName`: The name of Ceph Secret for `userId` to map RBD image. It must exist in the same namespace as PVCs. It is required.
+
 ### User provisioning requests
 
 Users request dynamically provisioned storage by including a storage class in their `PersistentVolumeClaim`.
@@ -152,6 +178,8 @@ In the future, the storage class may remain in an annotation or become a field o
 
 ### Sample output
 
+#### GCE
+
 This example uses GCE but any provisioner would follow the same flow.
 
 First we note there are no Persistent Volumes in the cluster.  After creating a storage class and a claim including that storage class, we see a new PV is created
@@ -184,6 +212,75 @@ $ kubectl get pv
 
 ```
 
+
+#### Ceph RBD
+
+First create Ceph admin's Secret in the system namespace. Here the Secret is created in `kube-system`:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/rbd/ceph-secret-admin.yaml --namespace=kube-system
+```
+
+Then create RBD Storage Class:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/rbd/rbd-storage-class.yaml
+```
+
+Before creating PVC in user's namespace (e.g. myns), make sure the Ceph user's Secret exists, if not, create the Secret:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/rbd/ceph-secret-user.yaml --namespace=myns
+```
+
+Now create a PVC in user's namespace (e.g. myns):
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/claim1.json --namespace=myns
+```
+
+Check the PV and PVC are created:
+
+```
+$ kubectl describe pvc --namespace=myns
+Name:		claim1
+Namespace:	myns
+Status:		Bound
+Volume:		pvc-1cfa23b3-664b-11e6-9eb9-90b11c09520d
+Labels:		<none>
+Capacity:	3Gi
+Access Modes:	RWO
+No events.
+
+$ kubectl describe pv
+Name:		pvc-1cfa23b3-664b-11e6-9eb9-90b11c09520d
+Labels:		<none>
+Status:		Bound
+Claim:		myns/claim1
+Reclaim Policy:	Delete
+Access Modes:	RWO
+Capacity:	3Gi
+Message:	
+Source:
+    Type:		RBD (a Rados Block Device mount on the host that shares a pod's lifetime)
+    CephMonitors:	[10.16.153.105:6789]
+    RBDImage:		kubernetes-dynamic-pvc-1cfb1862-664b-11e6-9a5d-90b11c09520d
+    FSType:		
+    RBDPool:		kube
+    RadosUser:		kube
+    Keyring:		/etc/ceph/keyring
+    SecretRef:		&{ceph-secret-user}
+    ReadOnly:		false
+No events.
+```
+
+Create a Pod to use the PVC:
+
+```
+$ kubectl create -f examples/experimental/persistent-volume-provisioning/rbd/pod.yaml --namespace=myns
+```
+
+
 <!-- BEGIN MUNGE: GENERATED_ANALYTICS -->
 [![Analytics](https://kubernetes-site.appspot.com/UA-36037335-10/GitHub/examples/experimental/persistent-volume-provisioning/README.md?pixel)]()
 <!-- END MUNGE: GENERATED_ANALYTICS -->
diff --git a/examples/experimental/persistent-volume-provisioning/rbd/ceph-secret-admin.yaml b/examples/experimental/persistent-volume-provisioning/rbd/ceph-secret-admin.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ceph-secret-admin
+data:
+  key: QVFEQ1pMdFhPUnQrSmhBQUFYaERWNHJsZ3BsMmNjcDR6RFZST0E9PQ==
diff --git a/examples/experimental/persistent-volume-provisioning/rbd/ceph-secret-user.yaml b/examples/experimental/persistent-volume-provisioning/rbd/ceph-secret-user.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: ceph-secret-user
+data:
+  key: QVFBTWdYaFZ3QkNlRGhBQTlubFBhRnlmVVNhdEdENGRyRldEdlE9PQ==
diff --git a/examples/experimental/persistent-volume-provisioning/rbd/pod.yaml b/examples/experimental/persistent-volume-provisioning/rbd/pod.yaml
@@ -0,0 +1,23 @@
+apiVersion: v1
+kind: ReplicationController
+metadata:
+  name: server
+spec:
+  replicas: 1
+  selector:
+    role: server
+  template:
+    metadata:
+      labels:
+        role: server
+    spec:
+      containers:
+      - name: server
+        image: nginx
+        volumeMounts:
+          - mountPath: /var/lib/www/html
+            name: mypvc
+      volumes:
+        - name: mypvc
+          persistentVolumeClaim:
+            claimName: claim1
diff --git a/examples/experimental/persistent-volume-provisioning/rbd/rbd-storage-class.yaml b/examples/experimental/persistent-volume-provisioning/rbd/rbd-storage-class.yaml
@@ -0,0 +1,14 @@
+apiVersion: extensions/v1beta1
+kind: StorageClass
+metadata:
+   name: slow
+provisioner: kubernetes.io/rbd
+parameters:
+    monitors: 10.16.153.105:6789
+    adminId: admin
+    adminSecretName: ceph-secret-admin
+    adminSecretNamespace: "kube-system"
+    pool: kube
+    userId: kube
+    userSecretName: ceph-secret-user
+
diff --git a/pkg/volume/rbd/disk_manager.go b/pkg/volume/rbd/disk_manager.go
@@ -26,6 +26,7 @@ import (
 	"os"
 
 	"github.com/golang/glog"
+	"k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/util/mount"
 	"k8s.io/kubernetes/pkg/volume"
 )
@@ -37,6 +38,10 @@ type diskManager interface {
 	AttachDisk(disk rbdMounter) error
 	// Detaches the disk from the kubelet's host machine.
 	DetachDisk(disk rbdUnmounter, mntPath string) error
+	// Creates a rbd image
+	CreateImage(provisioner *rbdVolumeProvisioner) (r *api.RBDVolumeSource, volumeSizeGB int, err error)
+	// Deletes a rbd image
+	DeleteImage(deleter *rbdVolumeDeleter) error
 }
 
 // utility to mount a disk based filesystem