New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Modify CephFs provisioner to use the ceph mgr commands #400
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,142 +17,118 @@ limitations under the License. | |
package cephfs | ||
|
||
import ( | ||
"fmt" | ||
"os" | ||
"path" | ||
"strconv" | ||
"strings" | ||
|
||
"github.com/ceph/ceph-csi/pkg/util" | ||
|
||
"k8s.io/klog" | ||
) | ||
|
||
const ( | ||
cephVolumesRoot = "csi-volumes" | ||
|
||
namespacePrefix = "ns-" | ||
namespacePrefix = "fsvolumens_" | ||
csiSubvolumeGroup = "csi" | ||
) | ||
|
||
func getCephRootPathLocal(volID volumeID) string { | ||
return fmt.Sprintf("%s/controller/volumes/root-%s", PluginFolder, string(volID)) | ||
} | ||
var ( | ||
// cephfsInit is used to create "csi" subvolume group for the first time the csi plugin loads. | ||
// Subvolume group create gets called every time the plugin loads, though it doesn't result in error | ||
// its unnecessary | ||
cephfsInit = false | ||
) | ||
|
||
func getCephRootVolumePathLocal(volID volumeID) string { | ||
return path.Join(getCephRootPathLocal(volID), cephVolumesRoot, string(volID)) | ||
} | ||
func getVolumeRootPathCeph(volOptions *volumeOptions, cr *util.Credentials, volID volumeID) (string, error) { | ||
stdout, _, err := util.ExecCommand( | ||
poornimag marked this conversation as resolved.
Show resolved
Hide resolved
|
||
"ceph", | ||
"fs", | ||
"subvolume", | ||
"getpath", | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I was expecting a There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Yes, we can do this, There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Aren't we breaking the abstraction if we let CSI construct subvolume path based on the subvolumegroup path? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The way I look at it the My intention for the base path is to avoid repeated calls to Ceph to fetch the same, as a result improving per operation performance by that much. |
||
volOptions.FsName, | ||
string(volID), | ||
"--group_name", | ||
csiSubvolumeGroup, | ||
"-m", volOptions.Monitors, | ||
"-c", util.CephConfigPath, | ||
"-n", cephEntityClientPrefix+cr.ID, | ||
"--key="+cr.Key) | ||
|
||
func getVolumeRootPathCeph(volID volumeID) string { | ||
return path.Join("/", cephVolumesRoot, string(volID)) | ||
if err != nil { | ||
klog.Errorf("failed to get the rootpath for the vol %s(%s)", string(volID), err) | ||
return "", err | ||
} | ||
return strings.TrimSuffix(string(stdout), "\n"), nil | ||
} | ||
|
||
func getVolumeNamespace(volID volumeID) string { | ||
return namespacePrefix + string(volID) | ||
} | ||
|
||
func setVolumeAttribute(root, attrName, attrValue string) error { | ||
return execCommandErr("setfattr", "-n", attrName, "-v", attrValue, root) | ||
} | ||
|
||
func createVolume(volOptions *volumeOptions, adminCr *util.Credentials, volID volumeID, bytesQuota int64) error { | ||
if err := mountCephRoot(volID, volOptions, adminCr); err != nil { | ||
return err | ||
} | ||
defer unmountCephRoot(volID) | ||
|
||
var ( | ||
volRoot = getCephRootVolumePathLocal(volID) | ||
volRootCreating = volRoot + "-creating" | ||
) | ||
|
||
if pathExists(volRoot) { | ||
klog.V(4).Infof("cephfs: volume %s already exists, skipping creation", volID) | ||
return nil | ||
} | ||
|
||
if err := createMountPoint(volRootCreating); err != nil { | ||
return err | ||
} | ||
|
||
if bytesQuota > 0 { | ||
if err := setVolumeAttribute(volRootCreating, "ceph.quota.max_bytes", fmt.Sprintf("%d", bytesQuota)); err != nil { | ||
func createVolume(volOptions *volumeOptions, cr *util.Credentials, volID volumeID, bytesQuota int64) error { | ||
//TODO: When we support multiple fs, need to hande subvolume group create for all fs's | ||
if !cephfsInit { | ||
err := execCommandErr( | ||
"ceph", | ||
"fs", | ||
"subvolumegroup", | ||
"create", | ||
volOptions.FsName, | ||
csiSubvolumeGroup, | ||
"--mode", | ||
"777", | ||
"--pool_layout", | ||
volOptions.Pool, | ||
"-m", volOptions.Monitors, | ||
"-c", util.CephConfigPath, | ||
"-n", cephEntityClientPrefix+cr.ID, | ||
"--key="+cr.Key) | ||
if err != nil { | ||
klog.Errorf("failed to create subvolume group csi, for the vol %s(%s)", string(volID), err) | ||
return err | ||
} | ||
klog.V(4).Infof("cephfs: created subvolume group csi") | ||
cephfsInit = true | ||
poornimag marked this conversation as resolved.
Show resolved
Hide resolved
|
||
} | ||
|
||
if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool", volOptions.Pool); err != nil { | ||
poornimag marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return fmt.Errorf("%v\ncephfs: Does pool '%s' exist?", err, volOptions.Pool) | ||
} | ||
|
||
if err := setVolumeAttribute(volRootCreating, "ceph.dir.layout.pool_namespace", getVolumeNamespace(volID)); err != nil { | ||
return err | ||
} | ||
|
||
if err := os.Rename(volRootCreating, volRoot); err != nil { | ||
return fmt.Errorf("couldn't mark volume %s as created: %v", volID, err) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func purgeVolume(volID volumeID, adminCr *util.Credentials, volOptions *volumeOptions) error { | ||
if err := mountCephRoot(volID, volOptions, adminCr); err != nil { | ||
err := execCommandErr( | ||
"ceph", | ||
"fs", | ||
"subvolume", | ||
"create", | ||
poornimag marked this conversation as resolved.
Show resolved
Hide resolved
|
||
volOptions.FsName, | ||
string(volID), | ||
strconv.FormatInt(bytesQuota, 10), | ||
"--group_name", | ||
csiSubvolumeGroup, | ||
"-m", volOptions.Monitors, | ||
"-c", util.CephConfigPath, | ||
"-n", cephEntityClientPrefix+cr.ID, | ||
"--key="+cr.Key) | ||
if err != nil { | ||
klog.Errorf("failed to create subvolume %s(%s) in fs %s", string(volID), err, volOptions.FsName) | ||
poornimag marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return err | ||
} | ||
defer unmountCephRoot(volID) | ||
|
||
var ( | ||
volRoot = getCephRootVolumePathLocal(volID) | ||
volRootDeleting = volRoot + "-deleting" | ||
) | ||
|
||
if pathExists(volRoot) { | ||
if err := os.Rename(volRoot, volRootDeleting); err != nil { | ||
return fmt.Errorf("couldn't mark volume %s for deletion: %v", volID, err) | ||
} | ||
} else { | ||
if !pathExists(volRootDeleting) { | ||
klog.V(4).Infof("cephfs: volume %s not found, assuming it to be already deleted", volID) | ||
return nil | ||
} | ||
} | ||
|
||
if err := os.RemoveAll(volRootDeleting); err != nil { | ||
return fmt.Errorf("failed to delete volume %s: %v", volID, err) | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func mountCephRoot(volID volumeID, volOptions *volumeOptions, adminCr *util.Credentials) error { | ||
cephRoot := getCephRootPathLocal(volID) | ||
|
||
// Root path is not set for dynamically provisioned volumes | ||
// Access to cephfs's / is required | ||
volOptions.RootPath = "/" | ||
|
||
if err := createMountPoint(cephRoot); err != nil { | ||
return err | ||
} | ||
|
||
m, err := newMounter(volOptions) | ||
func purgeVolume(volID volumeID, cr *util.Credentials, volOptions *volumeOptions) error { | ||
err := execCommandErr( | ||
"ceph", | ||
"fs", | ||
"subvolume", | ||
"rm", | ||
volOptions.FsName, | ||
string(volID), | ||
"--group_name", | ||
csiSubvolumeGroup, | ||
"--force", | ||
"-m", volOptions.Monitors, | ||
"-c", util.CephConfigPath, | ||
"-n", cephEntityClientPrefix+cr.ID, | ||
"--key="+cr.Key) | ||
if err != nil { | ||
return fmt.Errorf("failed to create mounter: %v", err) | ||
} | ||
|
||
if err = m.mount(cephRoot, adminCr, volOptions); err != nil { | ||
return fmt.Errorf("error mounting ceph root: %v", err) | ||
klog.Errorf("failed to purge subvolume %s(%s) in fs %s", string(volID), err, volOptions.FsName) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. avoid |
||
return err | ||
} | ||
|
||
poornimag marked this conversation as resolved.
Show resolved
Hide resolved
|
||
return nil | ||
} | ||
|
||
func unmountCephRoot(volID volumeID) { | ||
cephRoot := getCephRootPathLocal(volID) | ||
|
||
if err := unmountVolume(cephRoot); err != nil { | ||
klog.Errorf("failed to unmount %s with error %s", cephRoot, err) | ||
} else { | ||
if err := os.Remove(cephRoot); err != nil { | ||
klog.Errorf("failed to remove %s with error %s", cephRoot, err) | ||
} | ||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
please add a comment for this one
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
+1