ceph-volume: ensure correct --filestore/--bluestore behavior

src/ceph-volume/ceph_volume/devices/lvm/activate.py

@@ -19,6 +19,7 @@ def activate_filestore(lvs):
     if not osd_lv:
         raise RuntimeError('Unable to find a data LV for filestore activation')
     osd_id = osd_lv.tags['ceph.osd_id']
+    conf.cluster = osd_lv.tags['ceph.cluster_name']
     # it may have a volume with a journal
     osd_journal_lv = lvs.get(lv_tags={'ceph.type': 'journal'})
     # TODO: add sensible error reporting if this is ever the case
@@ -37,7 +38,7 @@ def activate_filestore(lvs):
     # mount the osd
     source = osd_lv.lv_path
     destination = '/var/lib/ceph/osd/%s-%s' % (conf.cluster, osd_id)
-    if not system.is_mounted(source, destination=destination):
+    if not system.device_is_mounted(source, destination=destination):
         process.run(['sudo', 'mount', '-v', source, destination])
 
     # always re-do the symlink regardless if it exists, so that the journal
@@ -84,6 +85,7 @@ def activate_bluestore(lvs):
     # find the osd
     osd_lv = lvs.get(lv_tags={'ceph.type': 'block'})
     osd_id = osd_lv.tags['ceph.osd_id']
+    conf.cluster = osd_lv.tags['ceph.cluster_name']
     osd_fsid = osd_lv.tags['ceph.osd_fsid']
     db_device_path = get_osd_device_path(osd_lv, lvs, 'db')
     wal_device_path = get_osd_device_path(osd_lv, lvs, 'wal')
@@ -93,13 +95,17 @@ def activate_bluestore(lvs):
     if not system.path_is_mounted(osd_path):
         # mkdir -p and mount as tmpfs
         prepare_utils.create_osd_path(osd_id, tmpfs=True)
-        # if the osd dir was not mounted via tmpfs, it means that the files are
-        # gone, so it needs to be 'primed' again. The command would otherwise
-        # fail if the directory was already populated
-        process.run([
-            'sudo', 'ceph-bluestore-tool',
-            'prime-osd-dir', '--dev', osd_lv.lv_path,
-            '--path', osd_path])
+    # XXX This needs to be removed once ceph-bluestore-tool can deal with
+    # symlinks that exist in the osd dir
+    for link_name in ['block', 'block.db', 'block.wal']:
+        link_path = os.path.join(osd_path, link_name)
+        if os.path.exists(link_path):
+            os.unlink(os.path.join(osd_path, link_name))
+    # Once symlinks are removed, the osd dir can be 'primed again.
+    process.run([
+        'sudo', 'ceph-bluestore-tool', '--cluster=%s' % conf.cluster,
+        'prime-osd-dir', '--dev', osd_lv.lv_path,
+        '--path', osd_path])
     # always re-do the symlink regardless if it exists, so that the block,
     # block.wal, and block.db devices that may have changed can be mapped
     # correctly every time

src/ceph-volume/ceph_volume/devices/lvm/common.py

@@ -32,12 +32,12 @@ def common_parser(prog, description):
     )
     parser.add_argument(
         '--bluestore',
-        action='store_true', default=True,
+        action='store_true',
         help='Use the bluestore objectstore',
     )
     parser.add_argument(
         '--filestore',
-        action='store_true', default=False,
+        action='store_true',
         help='Use the filestore objectstore',
     )
     parser.add_argument(

src/ceph-volume/ceph_volume/devices/lvm/create.py

@@ -50,4 +50,8 @@ def main(self):
             print(sub_command_help)
             return
         args = parser.parse_args(self.argv)
+        # Default to bluestore here since defaulting it in add_argument may
+        # cause both to be True
+        if args.bluestore is None and args.filestore is None:
+            args.bluestore = True
         self.create(args)

src/ceph-volume/ceph_volume/devices/lvm/prepare.py

@@ -58,18 +58,19 @@ def prepare_bluestore(block, wal, db, secrets, id_=None, fsid=None):
     osd_id = id_ or prepare_utils.create_id(fsid, json_secrets)
     # create the directory
     prepare_utils.create_osd_path(osd_id, tmpfs=True)
-    # symlink the block, wal, and db
+    # symlink the block
     prepare_utils.link_block(block, osd_id)
-    if wal:
-        prepare_utils.link_wal(wal, osd_id)
-    if db:
-        prepare_utils.link_db(db, osd_id)
     # get the latest monmap
     prepare_utils.get_monmap(osd_id)
     # write the OSD keyring if it doesn't exist already
     prepare_utils.write_keyring(osd_id, cephx_secret)
     # prepare the osd filesystem
-    prepare_utils.osd_mkfs_bluestore(osd_id, fsid, keyring=cephx_secret)
+    prepare_utils.osd_mkfs_bluestore(
+        osd_id, fsid,
+        keyring=cephx_secret,
+        wal=wal,
+        db=db
+    )
 
 
 class Prepare(object):
@@ -147,12 +148,14 @@ def prepare(self, args):
                 'ceph.osd_fsid': osd_fsid,
                 'ceph.osd_id': osd_id,
                 'ceph.cluster_fsid': cluster_fsid,
+                'ceph.cluster_name': conf.cluster,
                 'ceph.data_device': data_lv.lv_path,
                 'ceph.data_uuid': data_lv.lv_uuid,
             }
 
             journal_device, journal_uuid, tags = self.setup_device('journal', args.journal, tags)
 
+            tags['ceph.type'] = 'data'
             data_lv.set_tags(tags)
 
             prepare_filestore(
@@ -189,13 +192,15 @@ def prepare(self, args):
                 'ceph.osd_fsid': osd_fsid,
                 'ceph.osd_id': osd_id,
                 'ceph.cluster_fsid': cluster_fsid,
+                'ceph.cluster_name': conf.cluster,
                 'ceph.block_device': block_lv.lv_path,
                 'ceph.block_uuid': block_lv.lv_uuid,
             }
 
             wal_device, wal_uuid, tags = self.setup_device('wal', args.block_wal, tags)
             db_device, db_uuid, tags = self.setup_device('db', args.block_db, tags)
 
+            tags['ceph.type'] = 'block'
             block_lv.set_tags(tags)
 
             prepare_bluestore(
@@ -253,4 +258,8 @@ def main(self):
             print(sub_command_help)
             return
         args = parser.parse_args(self.argv)
+        # Default to bluestore here since defaulting it in add_argument may
+        # cause both to be True
+        if args.bluestore is None and args.filestore is None:
+            args.bluestore = True
         self.prepare(args)

src/ceph-volume/ceph_volume/util/prepare.py

@@ -183,8 +183,6 @@ def osd_mkfs_bluestore(osd_id, fsid, keyring=None, wal=False, db=False):
     """
     path = '/var/lib/ceph/osd/%s-%s/' % (conf.cluster, osd_id)
     monmap = os.path.join(path, 'activate.monmap')
-    wal_path = os.path.join(path, 'block.wal')
-    db_path = os.path.join(path, 'block.db')
 
     system.chown(path)
 
@@ -211,13 +209,15 @@ def osd_mkfs_bluestore(osd_id, fsid, keyring=None, wal=False, db=False):
 
     if wal:
         base_command.extend(
-            ['--bluestore-block-wal-path', wal_path]
+            ['--bluestore-block-wal-path', wal]
         )
+        system.chown(wal)
 
     if db:
         base_command.extend(
-            ['--bluestore-block-db-path', db_path]
+            ['--bluestore-block-db-path', db]
         )
+        system.chown(db)
 
     command = base_command + supplementary_command