Skip to content

Commit

Permalink
osd: support create osd with metadata partition
Browse files Browse the repository at this point in the history 
Currently, when rook provisions OSDs(in the OSD prepare job), rook effectively run a
c-v command such as the following.
```console
ceph-volume lvm batch --prepare <deviceA> <deviceB> <deviceC> --db-devices <metadataDevice>
```
but c-v lvm batch only supports disk and lvm, instead of disk partitions.

We can resort to `ceph-volume lvm prepare` to implement it.

Signed-off-by: Liang Zheng <zhengliang0901@gmail.com>
  • Loading branch information
@microyahoo
microyahoo committed Dec 15, 2023
1 parent 6449627 commit 1299eee
Show file tree
Hide file tree
Showing 2 changed files with 253 additions and 41 deletions.
125 changes: 86 additions & 39 deletions pkg/daemon/ceph/osd/volume.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@ const (
dbDeviceFlag = "--db-devices"
cephVolumeCmd = "ceph-volume"
cephVolumeMinDBSize = 1024 // 1GB

blockDBFlag = "--block.db"
blockDBSizeFlag = "--block.db-size"
dataFlag = "--data"
)

// These are not constants because they are used by the tests
Expand Down Expand Up @@ -606,6 +610,8 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *
osdsPerDeviceCount := sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice)
batchArgs := baseArgs

var hasMetadataPartition bool

metadataDevices := make(map[string]map[string]string)
for name, device := range devices.Entries {
if device.Data == -1 {
Expand Down Expand Up @@ -649,6 +655,9 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *
if metadataDevice.Type != sys.LVMType {
md = metadataDevice.Name
}
if metadataDevice.Type == sys.PartType {
hasMetadataPartition = true // ceph-volume lvm batch only supports disk and lvm
}

logger.Infof("using %s as metadataDevice for device %s and let ceph-volume lvm batch decide how to create volumes", md, deviceArg)
if _, ok := metadataDevices[md]; ok {
Expand Down Expand Up @@ -719,73 +728,111 @@ func (a *OsdAgent) initializeDevicesLVMMode(context *clusterd.Context, devices *
}
}

var prepareArgs []string
if hasMetadataPartition {
// ceph-volume lvm prepare --data {vg/lv} --block.wal {partition} --block.db {/path/to/device}
baseArgs := []string{"-oL", cephVolumeCmd, "--log-path", logPath, "lvm", "prepare", storeFlag}
if a.storeConfig.EncryptedDevice {
baseArgs = append(baseArgs, encryptedFlag)
}
prepareArgs = baseArgs
}

for md, conf := range metadataDevices {

mdArgs := batchArgs
if _, ok := conf["osdsperdevice"]; ok {
mdArgs = append(mdArgs, []string{
osdsPerDeviceFlag,
conf["osdsperdevice"],
}...)
if hasMetadataPartition {
mdArgs = prepareArgs
} else {
if _, ok := conf["osdsperdevice"]; ok {
mdArgs = append(mdArgs, []string{
osdsPerDeviceFlag,
conf["osdsperdevice"],
}...)
}
}
if _, ok := conf["deviceclass"]; ok {
mdArgs = append(mdArgs, []string{
crushDeviceClassFlag,
conf["deviceclass"],
}...)
}
if _, ok := conf["databasesizemb"]; ok {
if hasMetadataPartition {
devices := strings.Split(conf["devices"], " ")
if len(devices) > 1 {
logger.Warningf("device partition %s can only be used by one device", md)
}
mdArgs = append(mdArgs, []string{
databaseSizeFlag,
conf["databasesizemb"],
dataFlag,
devices[0],
}...)
if _, ok := conf["databasesizemb"]; ok {
mdArgs = append(mdArgs, []string{
blockDBSizeFlag,
conf["databasesizemb"],
}...)
}
} else {
if _, ok := conf["databasesizemb"]; ok {
mdArgs = append(mdArgs, []string{
databaseSizeFlag,
conf["databasesizemb"],
}...)
}
mdArgs = append(mdArgs, strings.Split(conf["devices"], " ")...)
}
mdArgs = append(mdArgs, strings.Split(conf["devices"], " ")...)

// Do not change device names if udev persistent names are passed
mdPath := md
if !strings.HasPrefix(mdPath, "/dev") {
mdPath = path.Join("/dev", md)
}

mdArgs = append(mdArgs, []string{
dbDeviceFlag,
mdPath,
}...)
if hasMetadataPartition {
mdArgs = append(mdArgs, []string{
blockDBFlag,
mdPath,
}...)
} else {
mdArgs = append(mdArgs, []string{
dbDeviceFlag,
mdPath,
}...)

// Reporting
reportArgs := append(mdArgs, []string{
"--report",
}...)
// Reporting
reportArgs := append(mdArgs, []string{
"--report",
}...)

if err := context.Executor.ExecuteCommand(baseCommand, reportArgs...); err != nil {
return errors.Wrap(err, "failed ceph-volume report") // fail return here as validation provided by ceph-volume
}
if err := context.Executor.ExecuteCommand(baseCommand, reportArgs...); err != nil {
return errors.Wrap(err, "failed ceph-volume report") // fail return here as validation provided by ceph-volume
}

reportArgs = append(reportArgs, []string{
"--format",
"json",
}...)
reportArgs = append(reportArgs, []string{
"--format",
"json",
}...)

cvOut, err := context.Executor.ExecuteCommandWithOutput(baseCommand, reportArgs...)
if err != nil {
return errors.Wrapf(err, "failed ceph-volume json report: %s", cvOut) // fail return here as validation provided by ceph-volume
}
cvOut, err := context.Executor.ExecuteCommandWithOutput(baseCommand, reportArgs...)
if err != nil {
return errors.Wrapf(err, "failed ceph-volume json report: %s", cvOut) // fail return here as validation provided by ceph-volume
}

logger.Debugf("ceph-volume reports: %+v", cvOut)
logger.Debugf("ceph-volume reports: %+v", cvOut)

var cvReports []cephVolReportV2
if err = json.Unmarshal([]byte(cvOut), &cvReports); err != nil {
return errors.Wrap(err, "failed to unmarshal ceph-volume report json")
}
var cvReports []cephVolReportV2
if err = json.Unmarshal([]byte(cvOut), &cvReports); err != nil {
return errors.Wrap(err, "failed to unmarshal ceph-volume report json")
}

if len(strings.Split(conf["devices"], " ")) != len(cvReports) {
return errors.Errorf("failed to create enough required devices, required: %s, actual: %v", cvOut, cvReports)
}
if len(strings.Split(conf["devices"], " ")) != len(cvReports) {
return errors.Errorf("failed to create enough required devices, required: %s, actual: %v", cvOut, cvReports)
}

for _, report := range cvReports {
if report.BlockDB != mdPath && !strings.HasSuffix(mdPath, report.BlockDB) {
return errors.Errorf("wrong db device for %s, required: %s, actual: %s", report.Data, mdPath, report.BlockDB)
for _, report := range cvReports {
if report.BlockDB != mdPath && !strings.HasSuffix(mdPath, report.BlockDB) {
return errors.Errorf("wrong db device for %s, required: %s, actual: %s", report.Data, mdPath, report.BlockDB)
}
}
}

Expand Down
169 changes: 167 additions & 2 deletions pkg/daemon/ceph/osd/volume_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -626,13 +626,23 @@ func TestConfigureCVDevices(t *testing.T) {
}

func testBaseArgs(args []string) error {
// stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm batch --prepare --bluestore --yes --osds-per-device 1 --crush-device-class hdd /dev/sda --db-devices /dev/sdl1 --report
if args[1] == "ceph-volume" && args[2] == "--log-path" && args[3] == "/tmp/ceph-log" && args[4] == "lvm" && args[5] == "batch" && args[6] == "--prepare" && args[7] == "--bluestore" && args[8] == "--yes" {
return nil
}

return errors.Errorf("unknown args %s ", args)
}

func testBasePrepareArgs(args []string) error {
// stdbuf -oL ceph-volume --log-path /tmp/ceph-log lvm prepare --bluestore --crush-device-class hdd --data /dev/sda --block.db /dev/sdj1
if args[1] == "ceph-volume" && args[2] == "--log-path" && args[3] == "/tmp/ceph-log" && args[4] == "lvm" && args[5] == "prepare" && args[6] == "--bluestore" {
return nil
}

return errors.Errorf("unknown args %s ", args)
}

func TestInitializeBlock(t *testing.T) {
os.Setenv(oposd.OSDStoreTypeVarName, "bluestore")
// Common vars for all the tests
Expand Down Expand Up @@ -832,6 +842,47 @@ func TestInitializeBlock(t *testing.T) {
}
}

// Test with metadata partition devices
{
devices := &DeviceOsdMapping{
Entries: map[string]*DeviceOsdIDEntry{
"sda": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/dev/sda", MetadataDevice: "sdb1"}},
},
}

executor := &exectest.MockExecutor{}
executor.MockExecuteCommand = func(command string, args ...string) error {
logger.Infof("%s %v", command, args)

// Validate base common args
err := testBasePrepareArgs(args)
if err != nil {
return err
}

if args[7] == "--data" && args[8] == "/dev/sda" && args[9] == "--block.db" && args[10] == "/dev/sdb1" {
return nil
}

return errors.Errorf("unknown command %s %s", command, args)
}

a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
context := &clusterd.Context{
Executor: executor,
Devices: []*sys.LocalDisk{
{Name: "sda"}, {Name: "sdb1", Type: sys.PartType},
},
}

err := a.initializeDevicesLVMMode(context, devices)
if err != nil {
assert.NoError(t, err, "failed metadata test")
} else {
logger.Info("success, go to next test")
}
}

// Test with metadata devices with dev by-id
{
metadataDeviceByIDPath := "/dev/disk/by-id/nvme-Samsung_SSD_970_EVO_Plus_1TB_XXX"
Expand Down Expand Up @@ -906,6 +957,58 @@ func TestInitializeBlock(t *testing.T) {
}
}

// Test with metadata partition devices with dev by-id
{
metadataDeviceByIDPath := "/dev/disk/by-id/nvme-BC511_NVMe_SK_hynix_512GB_CD08N413611008838-part1"
metadataDevicePath := "/dev/nvme0n1p1"
devices := &DeviceOsdMapping{
Entries: map[string]*DeviceOsdIDEntry{
"sda": {Data: -1, Metadata: nil, Config: DesiredDevice{Name: "/dev/sda", MetadataDevice: metadataDeviceByIDPath}},
},
}

executor := &exectest.MockExecutor{}
executor.MockExecuteCommand = func(command string, args ...string) error {
logger.Infof("%s %v", command, args)

// Validate base common args
err := testBasePrepareArgs(args)
if err != nil {
return err
}

if args[7] == "--data" && args[8] == "/dev/sda" && args[9] == "--block.db" && args[10] == metadataDevicePath {
return nil
}

return errors.Errorf("unknown command %s %s", command, args)
}

a := &OsdAgent{clusterInfo: &cephclient.ClusterInfo{CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0}}, nodeName: "node1", storeConfig: config.StoreConfig{StoreType: "bluestore"}}
context := &clusterd.Context{
Executor: executor,
Devices: []*sys.LocalDisk{
{
Name: "sda",
Type: "disk",
DevLinks: "/dev/disk/by-id/wwn-0x6f4ee080051fd00029bb505f1df6ee3a /dev/disk/by-path/pci-0000:3b:00.0-scsi-0:2:0:0",
},
{
Name: "nvme0n1p1",
Type: "part",
DevLinks: "/dev/disk/by-id/nvme-BC511_NVMe_SK_hynix_512GB_CD08N413611008838-part1 /dev/disk/by-partuuid/7cf003d3-3a56-4011-b736-b5a741b0aabc",
},
},
}

err := a.initializeDevicesLVMMode(context, devices)
if err != nil {
assert.NoError(t, err, "failed metadata partition device by-id test")
} else {
logger.Info("success, go to next test")
}
}

// Test with metadata devices with dev by-path
{
devices := &DeviceOsdMapping{
Expand Down Expand Up @@ -988,6 +1091,66 @@ func TestInitializeBlock(t *testing.T) {
logger.Info("success, go to next test")
}

// Test with metadata partition devices with dev by-path
{
devices := &DeviceOsdMapping{
Entries: map[string]*DeviceOsdIDEntry{
"sda": {
Data: -1,
Metadata: nil,
Config: DesiredDevice{
Name: "/dev/sda",
MetadataDevice: "/dev/disk/by-path/pci-0000:3a:00.0-nvme-1-part1",
},
},
},
}
metadataDevicePath := "/dev/nvme0n1p1"

executor := &exectest.MockExecutor{}
executor.MockExecuteCommand = func(command string, args ...string) error {
logger.Infof("%s %v", command, args)

// Validate base common args
err := testBasePrepareArgs(args)
if err != nil {
return err
}

if args[7] == "--data" && args[8] == "/dev/sda" && args[9] == "--block.db" && args[10] == metadataDevicePath {
return nil
}

return errors.Errorf("unknown command %s %s", command, args)
}

agent := &OsdAgent{
clusterInfo: &cephclient.ClusterInfo{
CephVersion: cephver.CephVersion{Major: 16, Minor: 2, Extra: 0},
},
nodeName: "node1",
storeConfig: config.StoreConfig{StoreType: "bluestore"},
}
context := &clusterd.Context{Executor: executor,
Devices: []*sys.LocalDisk{
{
Name: "sda",
Type: "disk",
DevLinks: "/dev/disk/by-id/wwn-0x6f4ee080051fd00029bb505f1df6ee3a /dev/disk/by-path/pci-0000:3b:00.0-scsi-0:2:0:0",
},
{
Name: "nvme0n1p1",
Type: "part",
DevLinks: "/dev/disk/by-path/pci-0000:3a:00.0-nvme-1-part1 /dev/disk/by-id/nvme-BC511_NVMe_SK_hynix_512GB_CD08N413611008838-part1 /dev/disk/by-partuuid/7cf003d3-3a56-4011-b736-b5a741b0aabc",
},
},
}

err := agent.initializeDevicesLVMMode(context, devices)
assert.NoError(t, err, "failed metadata device by-path test")
logger.Info("success, go to next test")
}

// Test with metadata devices with lvm
{
metadataDevicePath := "/dev/test-rook-vg/test-rook-lv"
Expand Down Expand Up @@ -1199,7 +1362,8 @@ func TestInitializeBlockPVC(t *testing.T) {
}
executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
logger.Infof("%s %v", command, args)
if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] == "--osd-id" && args[8] == "3" {
if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] == "--osd-id" && args[8] == "3" ||
args[1] == "ceph-volume" && args[4] == "raw" && args[5] == "prepare" && args[6] == "--bluestore" && args[9] == "--osd-id" && args[10] == "3" {
return initializeBlockPVCTestResult, nil
}

Expand All @@ -1220,7 +1384,8 @@ func TestInitializeBlockPVC(t *testing.T) {
}
executor.MockExecuteCommandWithCombinedOutput = func(command string, args ...string) (string, error) {
logger.Infof("%s %v", command, args)
if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] != "--osd-id" && args[8] != "3" {
if args[1] == "ceph-volume" && args[2] == "raw" && args[3] == "prepare" && args[4] == "--bluestore" && args[7] != "--osd-id" && args[8] != "3" ||
args[1] == "ceph-volume" && args[4] == "raw" && args[5] == "prepare" && args[6] == "--bluestore" && args[9] != "--osd-id" && args[10] != "3" {
return initializeBlockPVCTestResult, nil
}

Expand Down

0 comments on commit 1299eee

Please sign in to comment.