-
Notifications
You must be signed in to change notification settings - Fork 5.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
ceph-volume: work around phantom atari partitions
See Rook issue rook/rook#7940 for full information. Ceph bluestore disks can sometimes appear as though they have "phantom" Atari (AHDI) partitions created on them when they don't in reality. This is due to a series of bugs in the Linux kernel when it is built with Atari support enabled. This behavior does not appear for raw mode OSDs on partitions, only on disks. Changing the on-disk format of Bluestore OSDs comes with backwards-compatibility challenges, and fixing the issue in the Kernel could be years before users get a fix. Working around the Kernel issue in ceph-volume is therefore the best place to fix the issue for Ceph. To work around the issue in Ceph volume, there are two behaviors that need adjusted: 1. `ceph-volume inventory` should not report that a partition is available if the parent device is a BlueStore OSD. 2. `ceph-volume raw list` should report parent disks if the disk is a BlueStore OSD and not report the disk's children, BUT it should still report children if the parent disk is not a BlueStore OSD. Signed-off-by: Blaine Gardner <blaine.gardner@redhat.com>
- Loading branch information
Showing
5 changed files
with
351 additions
and
73 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
234 changes: 234 additions & 0 deletions
234
src/ceph-volume/ceph_volume/tests/devices/raw/test_list.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,234 @@ | ||
| import pytest | ||
| from mock.mock import patch | ||
| from ceph_volume.devices import raw | ||
|
|
||
| # Sample lsblk output is below that overviews the test scenario. (--json output for reader clarity) | ||
| # - sda and all its children are used for the OS | ||
| # - sdb is a bluestore OSD with phantom Atari partitions | ||
| # - sdc is an empty disk | ||
| # - sdd has 2 LVM device children | ||
| # > lsblk --paths --json | ||
| # { | ||
| # "blockdevices": [ | ||
| # {"name": "/dev/sda", "maj:min": "8:0", "rm": "0", "size": "128G", "ro": "0", "type": "disk", "mountpoint": null, | ||
| # "children": [ | ||
| # {"name": "/dev/sda1", "maj:min": "8:1", "rm": "0", "size": "487M", "ro": "0", "type": "part", "mountpoint": null}, | ||
| # {"name": "/dev/sda2", "maj:min": "8:2", "rm": "0", "size": "1.9G", "ro": "0", "type": "part", "mountpoint": null}, | ||
| # {"name": "/dev/sda3", "maj:min": "8:3", "rm": "0", "size": "125.6G", "ro": "0", "type": "part", "mountpoint": "/etc/hosts"} | ||
| # ] | ||
| # }, | ||
| # {"name": "/dev/sdb", "maj:min": "8:16", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null, | ||
| # "children": [ | ||
| # {"name": "/dev/sdb2", "maj:min": "8:18", "rm": "0", "size": "48G", "ro": "0", "type": "part", "mountpoint": null}, | ||
| # {"name": "/dev/sdb3", "maj:min": "8:19", "rm": "0", "size": "6M", "ro": "0", "type": "part", "mountpoint": null} | ||
| # ] | ||
| # }, | ||
| # {"name": "/dev/sdc", "maj:min": "8:32", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null}, | ||
| # {"name": "/dev/sdd", "maj:min": "8:48", "rm": "0", "size": "1T", "ro": "0", "type": "disk", "mountpoint": null, | ||
| # "children": [ | ||
| # {"name": "/dev/mapper/ceph--osd--block--1", "maj:min": "253:0", "rm": "0", "size": "512G", "ro": "0", "type": "lvm", "mountpoint": null}, | ||
| # {"name": "/dev/mapper/ceph--osd--block--2", "maj:min": "253:1", "rm": "0", "size": "512G", "ro": "0", "type": "lvm", "mountpoint": null} | ||
| # ] | ||
| # } | ||
| # ] | ||
| # } | ||
|
|
||
| def _devices_side_effect(): | ||
| return { | ||
| "/dev/sda": {}, | ||
| "/dev/sda1": {}, | ||
| "/dev/sda2": {}, | ||
| "/dev/sda3": {}, | ||
| "/dev/sdb": {}, | ||
| "/dev/sdb2": {}, | ||
| "/dev/sdb3": {}, | ||
| "/dev/sdc": {}, | ||
| "/dev/sdd": {}, | ||
| "/dev/mapper/ceph--osd--block--1": {}, | ||
| "/dev/mapper/ceph--osd--block--2": {}, | ||
| } | ||
|
|
||
| def _lsblk_list_output(): | ||
| return [ | ||
| '/dev/sda', | ||
| '/dev/sda1', | ||
| '/dev/sda2', | ||
| '/dev/sda3', | ||
| '/dev/sdb', | ||
| '/dev/sdb2', | ||
| '/dev/sdb3', | ||
| '/dev/sdc', | ||
| '/dev/sdd', | ||
| '/dev/mapper/ceph--osd--block--1', | ||
| '/dev/mapper/ceph--osd--block--2', | ||
| ] | ||
|
|
||
| # dummy lsblk output for device with optional parent output | ||
| def _lsblk_output(dev, parent=None): | ||
| if parent is None: | ||
| parent = "" | ||
| ret = 'NAME="{}" KNAME="{}" PKNAME="{}"'.format(dev, dev, parent) | ||
| return [ret] # needs to be in a list form | ||
|
|
||
| def _bluestore_tool_label_output_sdb(): | ||
| return '''{ | ||
| "/dev/sdb": { | ||
| "osd_uuid": "sdb-uuid", | ||
| "size": 1099511627776, | ||
| "btime": "2021-07-23T16:02:22.809186+0000", | ||
| "description": "main", | ||
| "bfm_blocks": "268435456", | ||
| "bfm_blocks_per_key": "128", | ||
| "bfm_bytes_per_block": "4096", | ||
| "bfm_size": "1099511627776", | ||
| "bluefs": "1", | ||
| "ceph_fsid": "sdb-fsid", | ||
| "kv_backend": "rocksdb", | ||
| "magic": "ceph osd volume v026", | ||
| "mkfs_done": "yes", | ||
| "osd_key": "AQAO6PpgK+y4CBAAixq/X7OVimbaezvwD/cDmg==", | ||
| "ready": "ready", | ||
| "require_osd_release": "16", | ||
| "whoami": "0" | ||
| } | ||
| }''' | ||
|
|
||
| def _bluestore_tool_label_output_sdb2(): | ||
| return '''{ | ||
| "/dev/sdb2": { | ||
| "osd_uuid": "sdb2-uuid", | ||
| "size": 1099511627776, | ||
| "btime": "2021-07-23T16:02:22.809186+0000", | ||
| "description": "main", | ||
| "bfm_blocks": "268435456", | ||
| "bfm_blocks_per_key": "128", | ||
| "bfm_bytes_per_block": "4096", | ||
| "bfm_size": "1099511627776", | ||
| "bluefs": "1", | ||
| "ceph_fsid": "sdb2-fsid", | ||
| "kv_backend": "rocksdb", | ||
| "magic": "ceph osd volume v026", | ||
| "mkfs_done": "yes", | ||
| "osd_key": "AQAO6PpgK+y4CBAAixq/X7OVimbaezvwD/cDmg==", | ||
| "ready": "ready", | ||
| "require_osd_release": "16", | ||
| "whoami": "2" | ||
| } | ||
| }''' | ||
|
|
||
| def _bluestore_tool_label_output_dm_okay(): | ||
| return '''{ | ||
| "/dev/mapper/ceph--osd--block--1": { | ||
| "osd_uuid": "lvm-1-uuid", | ||
| "size": 549751619584, | ||
| "btime": "2021-07-23T16:04:37.881060+0000", | ||
| "description": "main", | ||
| "bfm_blocks": "134216704", | ||
| "bfm_blocks_per_key": "128", | ||
| "bfm_bytes_per_block": "4096", | ||
| "bfm_size": "549751619584", | ||
| "bluefs": "1", | ||
| "ceph_fsid": "lvm-1-fsid", | ||
| "kv_backend": "rocksdb", | ||
| "magic": "ceph osd volume v026", | ||
| "mkfs_done": "yes", | ||
| "osd_key": "AQCU6Ppgz+UcIRAAh6IUjtPjiXBlEXfwO8ixzw==", | ||
| "ready": "ready", | ||
| "require_osd_release": "16", | ||
| "whoami": "2" | ||
| } | ||
| }''' | ||
|
|
||
| def _process_call_side_effect(command, **kw): | ||
| if "lsblk" in command: | ||
| if "/dev/" in command[-1]: | ||
| dev = command[-1] | ||
| if dev == "/dev/sda1" or dev == "/dev/sda2" or dev == "/dev/sda3": | ||
| return _lsblk_output(dev, parent="/dev/sda"), '', 0 | ||
| if dev == "/dev/sdb2" or dev == "/dev/sdb3": | ||
| return _lsblk_output(dev, parent="/dev/sdb"), '', 0 | ||
| if dev == "/dev/sda" or dev == "/dev/sdb" or dev == "/dev/sdc" or dev == "/dev/sdd": | ||
| return _lsblk_output(dev), '', 0 | ||
| if "mapper" in dev: | ||
| return _lsblk_output(dev, parent="/dev/sdd"), '', 0 | ||
| pytest.fail('dev {} needs behavior specified for it'.format(dev)) | ||
| if "/dev/" not in command: | ||
| return _lsblk_list_output(), '', 0 | ||
| pytest.fail('command {} needs behavior specified for it'.format(command)) | ||
|
|
||
| if "ceph-bluestore-tool" in command: | ||
| if "/dev/sdb" in command: | ||
| # sdb is a bluestore OSD | ||
| return _bluestore_tool_label_output_sdb(), '', 0 | ||
| if "/dev/sdb2" in command: | ||
| # sdb2 is a phantom atari partition that appears to have some valid bluestore info | ||
| return _bluestore_tool_label_output_sdb2(), '', 0 | ||
| if "/dev/mapper/ceph--osd--block--1" in command: | ||
| # dm device 1 is a valid bluestore OSD (the other is corrupted/invalid) | ||
| return _bluestore_tool_label_output_dm_okay(), '', 0 | ||
| # sda and children, sdb's children, sdc, sdd, dm device 2 all do NOT have bluestore OSD data | ||
| return [], 'fake No such file or directory error', 1 | ||
| pytest.fail('command {} needs behavior specified for it'.format(command)) | ||
|
|
||
| def _has_bluestore_label_side_effect(disk_path): | ||
| if "/dev/sda" in disk_path: | ||
| return False # disk and all children are for the OS | ||
| if disk_path == "/dev/sdb": | ||
| return True # sdb is a valid bluestore OSD | ||
| if disk_path == "/dev/sdb2": | ||
| return True # sdb2 appears to be a valid bluestore OSD even though it should not be | ||
| if disk_path == "/dev/sdc": | ||
| return False # empty disk | ||
| if disk_path == "/dev/sdd": | ||
| return False # has LVM subdevices | ||
| if disk_path == "/dev/mapper/ceph--osd--block--1": | ||
| return True # good OSD | ||
| if disk_path == "/dev/mapper/ceph--osd--block--2": | ||
| return False # corrupted | ||
| pytest.fail('device {} needs behavior specified for it'.format(disk_path)) | ||
|
|
||
| class TestList(object): | ||
|
|
||
| @patch('ceph_volume.util.device.disk.get_devices') | ||
| @patch('ceph_volume.util.disk.has_bluestore_label') | ||
| @patch('ceph_volume.process.call') | ||
| def test_raw_list(self, patched_call, patched_bluestore_label, patched_get_devices): | ||
| raw.list.logger.setLevel("DEBUG") | ||
| patched_call.side_effect = _process_call_side_effect | ||
| patched_bluestore_label.side_effect = _has_bluestore_label_side_effect | ||
| patched_get_devices.side_effect = _devices_side_effect | ||
|
|
||
| result = raw.list.List([]).generate() | ||
| assert len(result) == 2 | ||
|
|
||
| sdb = result['sdb-uuid'] | ||
| assert sdb['osd_uuid'] == 'sdb-uuid' | ||
| assert sdb['osd_id'] == 0 | ||
| assert sdb['device'] == '/dev/sdb' | ||
| assert sdb['ceph_fsid'] == 'sdb-fsid' | ||
| assert sdb['type'] == 'bluestore' | ||
|
|
||
| lvm1 = result['lvm-1-uuid'] | ||
| assert lvm1['osd_uuid'] == 'lvm-1-uuid' | ||
| assert lvm1['osd_id'] == 2 | ||
| assert lvm1['device'] == '/dev/mapper/ceph--osd--block--1' | ||
| assert lvm1['ceph_fsid'] == 'lvm-1-fsid' | ||
| assert lvm1['type'] == 'bluestore' | ||
|
|
||
| @patch('ceph_volume.util.device.disk.get_devices') | ||
| @patch('ceph_volume.util.disk.has_bluestore_label') | ||
| @patch('ceph_volume.process.call') | ||
| def test_raw_list_with_OSError(self, patched_call, patched_bluestore_label, patched_get_devices): | ||
| def _has_bluestore_label_side_effect_with_OSError(device_path): | ||
| if device_path == "/dev/sdd": | ||
| raise OSError('fake OSError') | ||
| return _has_bluestore_label_side_effect(device_path) | ||
|
|
||
| raw.list.logger.setLevel("DEBUG") | ||
| patched_call.side_effect = _process_call_side_effect | ||
| patched_bluestore_label.side_effect = _has_bluestore_label_side_effect_with_OSError | ||
| patched_get_devices.side_effect = _devices_side_effect | ||
|
|
||
| result = raw.list.List([]).generate() | ||
| assert len(result) == 1 | ||
| assert 'sdb-uuid' in result |
Oops, something went wrong.