From b04e35dab515b952670882988fdb7610a072ee8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20Han?= Date: Thu, 12 Dec 2019 19:06:22 +0100 Subject: [PATCH] ceph: add support for disk selection by full path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rook-Ceph now supports specify device by their fullpath instead of links created by the kernel like /dev/sdb. Closes: https://github.com/rook/rook/issues/1228 Co-authored-by: Sean Micklethwaite Signed-off-by: Sébastien Han --- Documentation/ceph-cluster-crd.md | 4 ++-- PendingReleaseNotes.md | 2 ++ cluster/examples/kubernetes/ceph/cluster.yaml | 1 + pkg/daemon/ceph/osd/daemon.go | 9 +++++++++ pkg/daemon/ceph/osd/daemon_test.go | 15 +++++++++++---- pkg/operator/ceph/cluster/osd/spec.go | 6 +++++- 6 files changed, 30 insertions(+), 7 deletions(-) diff --git a/Documentation/ceph-cluster-crd.md b/Documentation/ceph-cluster-crd.md index 772c87b5afe9..0033711a0f61 100755 --- a/Documentation/ceph-cluster-crd.md +++ b/Documentation/ceph-cluster-crd.md @@ -250,7 +250,7 @@ Below are the settings available, both at the cluster and individual node level, * `^/dev/sd.`: Selects all devices starting with `sd` * `^/dev/disk/by-path/pci-.*`: Selects all devices which are connected to PCI bus * `devices`: A list of individual device names belonging to this node to include in the storage cluster. - * `name`: The name of the device (e.g., `sda`) + * `name`: The name of the device (e.g., `sda`), or full udev path (e.g. `/dev/disk/by-id/ata-ST4000DM004-XXXX` - this will not change after reboots). * `config`: Device-specific config settings. See the [config settings](#osd-configuration-settings) below * `storageClassDeviceSets`: Explained in [Storage Class Device Sets](#storage-class-device-sets) @@ -442,7 +442,7 @@ spec: - name: "172.17.4.201" devices: # specific devices to use for storage can be specified for each node - name: "sdb" - - name: "sdc" + - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # both device name and explicit udev links are supported config: # configuration can be specified at the node level which overrides the cluster level config storeType: bluestore - name: "172.17.4.301" diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index c0febfaa06f5..475b7de6f222 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -13,6 +13,8 @@ - OSD on PVC now supports PVC expansion, if the size of the underlying block increases the Bluestore main block and the overall storage capacity will grow up. - Ceph Nautilus 14.2.5 is the minimum supported version - OSD on PVC doesn't use LVM anymore to configure OSD, but solely relies on the entire block device, done [here](https://github.com/rook/rook/pull/4435). +- Specific devices for OSDs can now be specified using the full udev path (e.g. /dev/disk/by-id/ata-ST4000DM004-XXXX) instead of the device name. + ### EdgeFS diff --git a/cluster/examples/kubernetes/ceph/cluster.yaml b/cluster/examples/kubernetes/ceph/cluster.yaml index fca3da25a2e4..a427c182750b 100644 --- a/cluster/examples/kubernetes/ceph/cluster.yaml +++ b/cluster/examples/kubernetes/ceph/cluster.yaml @@ -149,6 +149,7 @@ spec: # - name: "nvme01" # multiple osds can be created on high performance devices # config: # osdsPerDevice: "5" +# - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths # config: # configuration can be specified at the node level which overrides the cluster level config # storeType: filestore # - name: "172.17.4.301" diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index c491c73f3852..d30af6ffe8a1 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -380,6 +380,15 @@ func getAvailableDevices(context *clusterd.Context, desiredDevices []DesiredDevi } else if device.Name == desiredDevice.Name { logger.Infof("%q found in the desired devices", device.Name) matched = true + } else if strings.HasPrefix(desiredDevice.Name, "/dev/") { + devLinks := strings.Split(device.DevLinks, " ") + for _, link := range devLinks { + if link == desiredDevice.Name { + logger.Infof("%q found in the desired devices (matched by link: %q)", device.Name, link) + matched = true + break + } + } } matchedDevice = desiredDevice if matched { diff --git a/pkg/daemon/ceph/osd/daemon_test.go b/pkg/daemon/ceph/osd/daemon_test.go index 6153d48f9094..41a09d01d7d1 100644 --- a/pkg/daemon/ceph/osd/daemon_test.go +++ b/pkg/daemon/ceph/osd/daemon_test.go @@ -181,6 +181,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil {Name: "sdb", DevLinks: "/dev/disk/by-id/scsi-4567 /dev/disk/by-path/pci-4:5:6:7-scsi-1", RealName: "sdb"}, {Name: "sdc", DevLinks: "/dev/disk/by-id/scsi-89ab /dev/disk/by-path/pci-8:9:a:b-scsi-1", RealName: "sdc"}, {Name: "sdd", DevLinks: "/dev/disk/by-id/scsi-cdef /dev/disk/by-path/pci-c:d:e:f-scsi-1", RealName: "sdd"}, + {Name: "sde", DevLinks: "/dev/disk/by-id/sde-0x0000 /dev/disk/by-path/pci-0000:00:18.0-ata-1", RealName: "sde"}, {Name: "nvme01", DevLinks: "/dev/disk/by-id/nvme-0246 /dev/disk/by-path/pci-0:2:4:6-nvme-1", RealName: "nvme01"}, {Name: "rda", RealName: "rda"}, {Name: "rdb", RealName: "rdb"}, @@ -194,7 +195,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil pvcBackedOSD := false mapping, err := getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version) assert.Nil(t, err) - assert.Equal(t, 6, len(mapping.Entries)) + assert.Equal(t, 7, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) assert.Equal(t, -1, mapping.Entries["rda"].Data) @@ -207,7 +208,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil version = cephver.Nautilus mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "nvme01", pvcBackedOSD, version) assert.Nil(t, err) - assert.Equal(t, 5, len(mapping.Entries)) + assert.Equal(t, 6, len(mapping.Entries)) // Do not skip partition anymore version = cephver.Octopus @@ -224,7 +225,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil // select the sd* devices mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^sd.$", IsFilter: true}}, "", pvcBackedOSD, version) assert.Nil(t, err) - assert.Equal(t, 2, len(mapping.Entries)) + assert.Equal(t, 3, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) @@ -245,7 +246,7 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil // select the sd* devices by path names mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "^/dev/sd.$", IsDevicePathFilter: true}}, "", pvcBackedOSD, version) assert.Nil(t, err) - assert.Equal(t, 2, len(mapping.Entries)) + assert.Equal(t, 3, len(mapping.Entries)) assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) @@ -256,6 +257,12 @@ NAME="sdb1" SIZE="30" TYPE="part" PKNAME="sdb"`, nil assert.Equal(t, -1, mapping.Entries["sda"].Data) assert.Equal(t, -1, mapping.Entries["sdd"].Data) + // select a device by explicit link + mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "/dev/disk/by-id/sde-0x0000"}}, "", pvcBackedOSD, version) + assert.Nil(t, err) + assert.Equal(t, 1, len(mapping.Entries)) + assert.Equal(t, -1, mapping.Entries["sde"].Data) + // test on PVC pvcBackedOSD = true mapping, err = getAvailableDevices(context, []DesiredDevice{{Name: "all"}}, "", pvcBackedOSD, version) diff --git a/pkg/operator/ceph/cluster/osd/spec.go b/pkg/operator/ceph/cluster/osd/spec.go index 46f748a7f503..aa0582ed47b3 100644 --- a/pkg/operator/ceph/cluster/osd/spec.go +++ b/pkg/operator/ceph/cluster/osd/spec.go @@ -871,7 +871,11 @@ func (c *Cluster) provisionOSDContainer(osdProps osdProperties, copyBinariesMoun } else { devSuffix += ":" } - deviceNames[i] = device.Name + devSuffix + deviceID := device.Name + if device.FullPath != "" { + deviceID = device.FullPath + } + deviceNames[i] = deviceID + devSuffix } envVars = append(envVars, dataDevicesEnvVar(strings.Join(deviceNames, ","))) devMountNeeded = true