Permalink
Browse files

Merge pull request #10135 from david-z/wip-enhance-ceph-disk-bluestore

ceph-disk: support creating block.db and block.wal with customized size for bluestore

Reviewed-by: Loic Dachary <ldachary@redhat.com>
  • Loading branch information...
2 parents b846597 + 67b11b0 commit eb968f886e02fa683b0d3965e71d1ead2677a059 @dachary dachary committed on GitHub Sep 22, 2016
Showing with 312 additions and 24 deletions.
  1. +83 −0 qa/workunits/ceph-disk/ceph-disk-test.py
  2. +151 −24 src/ceph-disk/ceph_disk/main.py
  3. +78 −0 udev/95-ceph-osd.rules
@@ -143,6 +143,15 @@ def get_osd_partition(self, uuid):
def get_journal_partition(self, uuid):
return self.get_space_partition('journal', uuid)
+ def get_block_partition(self, uuid):
+ return self.get_space_partition('block', uuid)
+
+ def get_blockdb_partition(self, uuid):
+ return self.get_space_partition('block.db', uuid)
+
+ def get_blockwal_partition(self, uuid):
+ return self.get_space_partition('block.wal', uuid)
+
def get_space_partition(self, name, uuid):
data_partition = self.get_osd_partition(uuid)
space_dev = data_partition[name + '_dev']
@@ -454,10 +463,84 @@ def test_activate_bluestore(self):
" " + disk)
c.wait_for_osd_up(osd_uuid)
device = json.loads(c.sh("ceph-disk list --format json " + disk))[0]
+ assert len(device['partitions']) == 4
+ c.check_osd_status(osd_uuid, 'block')
+ c.check_osd_status(osd_uuid, 'block.wal')
+ c.check_osd_status(osd_uuid, 'block.db')
+ c.helper("pool_read_write")
+ c.destroy_osd(osd_uuid)
+ c.sh("ceph-disk --verbose zap " + disk)
+
+ def test_activate_bluestore_seperated_block_db_wal(self):
+ c = CephDisk()
+ disk1 = c.unused_disks()[0]
+ disk2 = c.unused_disks()[1]
+ osd_uuid = str(uuid.uuid1())
+ c.sh("ceph-disk --verbose zap " + disk1 + " " + disk2)
+ c.conf['global']['osd objectstore'] = 'bluestore'
+ c.save_conf()
+ c.sh("ceph-disk --verbose prepare --bluestore --osd-uuid " + osd_uuid +
+ " " + disk1 + " --block.db " + disk2 + " --block.wal " + disk2)
+ c.wait_for_osd_up(osd_uuid)
+ device = json.loads(c.sh("ceph-disk list --format json " + disk1))[0]
+ assert len(device['partitions']) == 2
+ device = json.loads(c.sh("ceph-disk list --format json " + disk2))[0]
assert len(device['partitions']) == 2
c.check_osd_status(osd_uuid, 'block')
+ c.check_osd_status(osd_uuid, 'block.wal')
+ c.check_osd_status(osd_uuid, 'block.db')
c.helper("pool_read_write")
c.destroy_osd(osd_uuid)
+ c.sh("ceph-disk --verbose zap " + disk1 + " " + disk2)
+
+ def test_activate_bluestore_reuse_db_wal_partition(self):
+ c = CephDisk()
+ disks = c.unused_disks()
+ block_disk = disks[0]
+ db_wal_disk = disks[1]
+ #
+ # Create an OSD with two disks (one for block,
+ # the other for block.db and block.wal ) and then destroy osd.
+ #
+ osd_uuid1 = str(uuid.uuid1())
+ c.sh("ceph-disk --verbose zap " + block_disk + " " + db_wal_disk)
+ c.conf['global']['osd objectstore'] = 'bluestore'
+ c.save_conf()
+ c.sh("ceph-disk --verbose prepare --bluestore --osd-uuid " +
+ osd_uuid1 + " " + block_disk + " --block.db " + db_wal_disk +
+ " --block.wal " + db_wal_disk)
+ c.wait_for_osd_up(osd_uuid1)
+ blockdb_partition = c.get_blockdb_partition(osd_uuid1)
+ blockdb_path = blockdb_partition['path']
+ blockwal_partition = c.get_blockwal_partition(osd_uuid1)
+ blockwal_path = blockwal_partition['path']
+ c.destroy_osd(osd_uuid1)
+ c.sh("ceph-disk --verbose zap " + block_disk)
+ #
+ # Create another OSD with the block.db and block.wal partition
+ # of the previous OSD
+ #
+ osd_uuid2 = str(uuid.uuid1())
+ c.sh("ceph-disk --verbose prepare --bluestore --osd-uuid " +
+ osd_uuid2 + " " + block_disk + " --block.db " + blockdb_path +
+ " --block.wal " + blockwal_path)
+ c.wait_for_osd_up(osd_uuid2)
+ device = json.loads(c.sh("ceph-disk list --format json " + block_disk))[0]
+ assert len(device['partitions']) == 2
+ device = json.loads(c.sh("ceph-disk list --format json " + db_wal_disk))[0]
+ assert len(device['partitions']) == 2
+ c.check_osd_status(osd_uuid2, 'block')
+ c.check_osd_status(osd_uuid2, 'block.wal')
+ c.check_osd_status(osd_uuid2, 'block.db')
+ blockdb_partition = c.get_blockdb_partition(osd_uuid2)
+ blockwal_partition = c.get_blockwal_partition(osd_uuid2)
+ #
+ # Verify the previous OSD partition has been reused
+ #
+ assert blockdb_partition['path'] == blockdb_path
+ assert blockwal_partition['path'] == blockwal_path
+ c.destroy_osd(osd_uuid2)
+ c.sh("ceph-disk --verbose zap " + block_disk + " " + db_wal_disk)
def test_activate_with_journal_dev_is_symlink(self):
c = CephDisk()
@@ -57,6 +57,16 @@
'ready': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
'tobe': 'cafecafe-9b03-4f30-b4c6-b4b80ceff106',
},
+ 'block.db': {
+ # identical because creating a block is atomic
+ 'ready': '30cd0809-c2b2-499c-8879-2d6b78529876',
+ 'tobe': '30cd0809-c2b2-499c-8879-2d6b785292be',
+ },
+ 'block.wal': {
+ # identical because creating a block is atomic
+ 'ready': '5ce17fce-4087-4169-b7ff-056cc58473f9',
+ 'tobe': '5ce17fce-4087-4169-b7ff-056cc58472be',
+ },
'osd': {
'ready': '4fbd7e29-9d25-41b8-afd0-062c0ceff05d',
'tobe': '89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be',
@@ -75,6 +85,14 @@
'ready': 'cafecafe-9b03-4f30-b4c6-35865ceff106',
'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
},
+ 'block.db': {
+ 'ready': '166418da-c469-4022-adf4-b30afd37f176',
+ 'tobe': '7521c784-4626-4260-bc8d-ba77a0f5f2be',
+ },
+ 'block.wal': {
+ 'ready': '86a32090-3647-40b9-bbbd-38d8c573aa86',
+ 'tobe': '92dad30f-175b-4d40-a5b0-5c0a258b42be',
+ },
'osd': {
'ready': '4fbd7e29-9d25-41b8-afd0-35865ceff05d',
'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
@@ -89,6 +107,14 @@
'ready': 'cafecafe-9b03-4f30-b4c6-5ec00ceff106',
'tobe': '89c57f98-2fe5-4dc0-89c1-35865ceff2be',
},
+ 'block.db': {
+ 'ready': '93b0052d-02d9-4d8a-a43b-33a3ee4dfbc3',
+ 'tobe': '69d17c68-3e58-4399-aff0-b68265f2e2be',
+ },
+ 'block.wal': {
+ 'ready': '306e8683-4fe2-4330-b7c0-00a917c16966',
+ 'tobe': 'f2d89683-a621-4063-964a-eb1f7863a2be',
+ },
'osd': {
'ready': '4fbd7e29-9d25-41b8-afd0-5ec00ceff05d',
'tobe': '89c57f98-2fe5-4dc0-89c1-5ec00ceff2be',
@@ -103,6 +129,14 @@
'ready': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
'tobe': 'cafecafe-8ae0-4982-bf9d-5a8d867af560',
},
+ 'block.db': {
+ 'ready': 'ec6d6385-e346-45dc-be91-da2a7c8b3261',
+ 'tobe': 'ec6d6385-e346-45dc-be91-da2a7c8b32be',
+ },
+ 'block.wal': {
+ 'ready': '01b41e1b-002a-453c-9f17-88793989ff8f',
+ 'tobe': '01b41e1b-002a-453c-9f17-88793989f2be',
+ },
'osd': {
'ready': '4fbd7e29-8ae0-4982-bf9d-5a8d867af560',
'tobe': '89c57f98-8ae0-4982-bf9d-5a8d867af560',
@@ -1805,6 +1839,8 @@ def __init__(self, args):
self.lockbox = Lockbox(args)
self.data = PrepareBluestoreData(args)
self.block = PrepareBluestoreBlock(args)
+ self.blockdb = PrepareBluestoreBlockDB(args)
+ self.blockwal = PrepareBluestoreBlockWAL(args)
@staticmethod
def parser():
@@ -1821,17 +1857,19 @@ def parent_parsers():
return [
PrepareBluestore.parser(),
PrepareBluestoreBlock.parser(),
+ PrepareBluestoreBlockDB.parser(),
+ PrepareBluestoreBlockWAL.parser(),
]
def prepare_locked(self):
if self.data.args.dmcrypt:
self.lockbox.prepare()
- self.data.prepare(self.block)
+ self.data.prepare(self.blockdb, self.blockwal, self.block)
class Space(object):
- NAMES = ('block', 'journal')
+ NAMES = ('block', 'journal', 'block.db', 'block.wal')
class PrepareSpace(object):
@@ -1903,7 +1941,7 @@ def is_device(self):
return self.type == self.DEVICE
@staticmethod
- def parser(name):
+ def parser(name, positional=True):
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument(
'--%s-uuid' % name,
@@ -1920,13 +1958,15 @@ def parser(name):
action='store_true', default=None,
help='verify that %s is a block device' % name.upper(),
)
- parser.add_argument(
- name,
- metavar=name.upper(),
- nargs='?',
- help=('path to OSD %s disk block device;' % name +
- ' leave out to store %s in file' % name),
- )
+
+ if positional:
+ parser.add_argument(
+ name,
+ metavar=name.upper(),
+ nargs='?',
+ help=('path to OSD %s disk block device;' % name +
+ ' leave out to store %s in file' % name),
+ )
return parser
def wants_space(self):
@@ -2068,17 +2108,18 @@ def prepare_device(self):
partition.format()
partition.map()
- command_check_call(
- [
- 'sgdisk',
- '--typecode={num}:{uuid}'.format(
- num=num,
- uuid=partition.ptype_for_name(self.name),
- ),
- '--',
- getattr(self.args, self.name),
- ],
- )
+ command_check_call(
+ [
+ 'sgdisk',
+ '--typecode={num}:{uuid}'.format(
+ num=num,
+ uuid=partition.ptype_for_name(self.name),
+ ),
+ '--',
+ getattr(self.args, self.name),
+ ],
+ )
+ update_partition(getattr(self.args, self.name), 'prepared')
LOG.debug('%s is GPT partition %s',
self.name.capitalize(),
@@ -2128,7 +2169,15 @@ def __init__(self, args):
super(PrepareBluestoreBlock, self).__init__(args)
def get_space_size(self):
- return 0 # get as much space as possible
+ block_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_size',
+ )
+
+ if block_size is None:
+ return 0 # get as much space as possible
+ else:
+ return int(block_size) / 1048576 # MB
def desired_partition_number(self):
if self.args.block == self.args.data:
@@ -2142,6 +2191,76 @@ def parser():
return PrepareSpace.parser('block')
+class PrepareBluestoreBlockDB(PrepareSpace):
+
+ def __init__(self, args):
+ self.name = 'block.db'
+ super(PrepareBluestoreBlockDB, self).__init__(args)
+
+ def get_space_size(self):
+ block_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_db_size',
+ )
+
+ if block_size is None:
+ return 64 # MB, default value
+ else:
+ return int(block_size) / 1048576 # MB
+
+ def desired_partition_number(self):
+ if getattr(self.args, 'block.db') == self.args.data:
+ num = 3
+ else:
+ num = 0
+ return num
+
+ @staticmethod
+ def parser():
+ parser = PrepareSpace.parser('block.db', positional=False)
+ parser.add_argument(
+ '--block.db',
+ metavar='BLOCKDB',
+ help='path to the device or file for bluestore block.db',
+ )
+ return parser
+
+
+class PrepareBluestoreBlockWAL(PrepareSpace):
+
+ def __init__(self, args):
+ self.name = 'block.wal'
+ super(PrepareBluestoreBlockWAL, self).__init__(args)
+
+ def get_space_size(self):
+ block_size = get_conf(
+ cluster=self.args.cluster,
+ variable='bluestore_block_wal_size',
+ )
+
+ if block_size is None:
+ return 128 # MB, default value
+ else:
+ return int(block_size) / 1048576 # MB
+
+ def desired_partition_number(self):
+ if getattr(self.args, 'block.wal') == self.args.data:
+ num = 4
+ else:
+ num = 0
+ return num
+
+ @staticmethod
+ def parser():
+ parser = PrepareSpace.parser('block.wal', positional=False)
+ parser.add_argument(
+ '--block.wal',
+ metavar='BLOCKWAL',
+ help='path to the device or file for bluestore block.wal',
+ )
+ return parser
+
+
class CryptHelpers(object):
@staticmethod
@@ -4362,7 +4481,11 @@ def main_trigger(args):
)
elif parttype in (PTYPE['regular']['block']['ready'],
- PTYPE['mpath']['block']['ready']):
+ PTYPE['regular']['block.db']['ready'],
+ PTYPE['regular']['block.wal']['ready'],
+ PTYPE['mpath']['block']['ready'],
+ PTYPE['mpath']['block.db']['ready'],
+ PTYPE['mpath']['block.wal']['ready']):
out, err, ret = command(
ceph_disk +
[
@@ -4372,7 +4495,11 @@ def main_trigger(args):
)
elif parttype in (PTYPE['plain']['block']['ready'],
- PTYPE['luks']['block']['ready']):
+ PTYPE['plain']['block.db']['ready'],
+ PTYPE['plain']['block.wal']['ready'],
+ PTYPE['luks']['block']['ready'],
+ PTYPE['luks']['block.db']['ready'],
+ PTYPE['luks']['block.wal']['ready']):
out, err, ret = command(
ceph_disk +
[
Oops, something went wrong.

0 comments on commit eb968f8

Please sign in to comment.