diff --git a/Makefile.am b/Makefile.am index adeb4e577289b..03cb914079fe9 100644 --- a/Makefile.am +++ b/Makefile.am @@ -10,7 +10,8 @@ EXTRA_DIST += \ src/test/cli \ src/test/downloads \ udev/50-rbd.rules \ - udev/95-ceph-osd.rules + udev/95-ceph-osd.rules \ + udev/95-ceph-osd-alt.rules all-local: diff --git a/ceph.spec.in b/ceph.spec.in index a2b504d2d2fb4..023258cbe6178 100644 --- a/ceph.spec.in +++ b/ceph.spec.in @@ -25,10 +25,11 @@ Requires: librbd1 = %{version}-%{release} Requires: librados2 = %{version}-%{release} Requires: libcephfs1 = %{version}-%{release} Requires: python +Requires: python-argparse +Requires: python-lockfile Requires: cryptsetup Requires: parted Requires: util-linux -Requires: python-argparse Requires: hdparm Requires(post): binutils BuildRoot: %{_tmppath}/%{name}-%{version}-build @@ -301,7 +302,11 @@ chmod 0644 $RPM_BUILD_ROOT%{_docdir}/ceph/sample.fetch_config # udev rules install -m 0644 -D udev/50-rbd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/50-rbd.rules +%if 0%{?centos} +install -m 0644 -D udev/95-ceph-osd-alt.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%else install -m 0644 -D udev/95-ceph-osd.rules $RPM_BUILD_ROOT/lib/udev/rules.d/95-ceph-osd.rules +%endif #set up placeholder directories mkdir -p $RPM_BUILD_ROOT%{_sysconfdir}/ceph @@ -386,6 +391,7 @@ fi %{_sbindir}/ceph-disk %{_sbindir}/ceph-disk-activate %{_sbindir}/ceph-disk-prepare +%{_sbindir}/ceph-disk-udev %{_sbindir}/ceph-create-keys %{_sbindir}/rcceph /sbin/mkcephfs diff --git a/debian/control b/debian/control index 98d0b37ffc1e9..efa86377c52ed 100644 --- a/debian/control +++ b/debian/control @@ -45,6 +45,8 @@ Depends: binutils, gdisk, parted, python, + python-argparse, + python-lockfile, sdparm | hdparm, uuid-runtime, xfsprogs, diff --git a/src/Makefile.am b/src/Makefile.am index d528b78a1bef1..0b223f2029933 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -33,6 +33,7 @@ ceph_sbin_SCRIPTS = \ ceph-disk \ ceph-disk-prepare \ ceph-disk-activate \ + ceph-disk-udev \ ceph-create-keys sbin_SCRIPTS = \ @@ -1192,6 +1193,7 @@ EXTRA_DIST += \ ceph-disk \ ceph-disk-prepare \ ceph-disk-activate \ + ceph-disk-udev \ ceph-create-keys \ mount.fuse.ceph diff --git a/src/ceph-disk b/src/ceph-disk index 68a171938c594..c9515997ded46 100755 --- a/src/ceph-disk +++ b/src/ceph-disk @@ -12,6 +12,7 @@ import stat import sys import tempfile import uuid +import lockfile CEPH_OSD_ONDISK_MAGIC = 'ceph osd volume v026' @@ -62,6 +63,7 @@ if LOG_NAME == '__main__': LOG_NAME = os.path.basename(sys.argv[0]) LOG = logging.getLogger(LOG_NAME) +lock = lockfile.FileLock('/var/lib/ceph/tmp/ceph-disk.lock') ###### exceptions ######## @@ -683,6 +685,16 @@ def zap(dev): raise Error(e) +def get_udev_version(): + version = _check_output( + args=[ + 'udevadm', + '--version', + ], + ) + return int(version) + + def prepare_journal_dev( data, journal, @@ -761,9 +773,24 @@ def prepare_journal_dev( ], ) - journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format( - journal_uuid=journal_uuid, - ) + if get_udev_version() >= 172: + journal_symlink = '/dev/disk/by-partuuid/{journal_uuid}'.format( + journal_uuid=journal_uuid, + ) + else: + # udev prior to version 172 doesn't create by-partuuid directory + # use by-path symlink instead. This is the third symlink returned + # by udevadm when queried. + LOG.debug('Using alternate persistant name for journal symlink') + symlinks = _check_output( + args=[ + 'udevadm', + 'info', + '--query=symlink', + '--name={name}'.format(name=os.path.basename(journal)), + ], + ) + journal_symlink='/dev/{symlink}-part{num}'.format(symlink=symlinks.split()[2], num=num) journal_dmcrypt = None if journal_dm_keypath: @@ -1035,6 +1062,7 @@ def main_prepare(args): osd_dm_keypath = None try: + lock.acquire() if not os.path.exists(args.data): raise Error('data path does not exist', args.data) @@ -1163,12 +1191,14 @@ def main_prepare(args): ) else: raise Error('not a dir or block device', args.data) + lock.release() except Error as e: if journal_dm_keypath: os.unlink(journal_dm_keypath) if osd_dm_keypath: os.unlink(osd_dm_keypath) + lock.release() raise e @@ -1559,26 +1589,32 @@ def main_activate(args): if not os.path.exists(args.path): raise Error('%s does not exist', args.path) - mode = os.stat(args.path).st_mode - if stat.S_ISBLK(mode): - (cluster, osd_id) = mount_activate( - dev=args.path, - activate_key_template=args.activate_key_template, - init=args.mark_init, - ) - elif stat.S_ISDIR(mode): - (cluster, osd_id) = activate_dir( - path=args.path, - activate_key_template=args.activate_key_template, - init=args.mark_init, + lock.acquire() + try: + mode = os.stat(args.path).st_mode + if stat.S_ISBLK(mode): + (cluster, osd_id) = mount_activate( + dev=args.path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + elif stat.S_ISDIR(mode): + (cluster, osd_id) = activate_dir( + path=args.path, + activate_key_template=args.activate_key_template, + init=args.mark_init, + ) + else: + raise Error('%s is not a directory or block device', args.path) + + start_daemon( + cluster=cluster, + osd_id=osd_id, ) - else: - raise Error('%s is not a directory or block device', args.path) + lock.release() - start_daemon( - cluster=cluster, - osd_id=osd_id, - ) + except: + lock.release() diff --git a/src/ceph-disk-udev b/src/ceph-disk-udev new file mode 100755 index 0000000000000..a96c21652aa99 --- /dev/null +++ b/src/ceph-disk-udev @@ -0,0 +1,56 @@ +#! /bin/sh + +# Wrapper for the ceph udev rules. Since older versions of udev do not support gpt label fields, this shell +# script is invoked from the udev rule to read the needed gpt label fields and call the appropriate ceph +# OSD functions. + +PARTNO=$1 +NAME=$2 +PARENT_NAME=$3 + +# Get GPT partition type guid +ID_PART_ENTRY_TYPE=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition GUID code" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') +case $ID_PART_ENTRY_TYPE in + +45b0969e-9b03-4f30-b4c6-b4b80ceff106) + # JOURNAL_UUID + ;; + +45b0969e-9b03-4f30-b4c6-5ec00ceff106) + # DMCRYPT_JOURNAL_UUID + # Map journal if using dm-crypt + ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} + ;; + +4fbd7e29-9d25-41b8-afd0-062c0ceff05d) + # OSD_UUID + # activate ceph-tagged partitions. + /usr/sbin/ceph-disk -v activate --mount /dev/${NAME} + ;; + +4fbd7e29-9d25-41b8-afd0-5ec00ceff05d) + # DMCRYPT_OSD_UUID + # Map data device and activate ceph-tagged partitions + # for dm-crypted data devices + ID_PART_ENTRY_UUID=$(/usr/sbin/sgdisk --info=${PARTNO} /dev/${PARENT_NAME} | grep "Partition unique GUID" | awk '{print $4}' | tr '[:upper:]' '[:lower:]') + /sbin/cryptsetup --key-file /etc/ceph/dmcrypt-keys/${ID_PART_ENTRY_UUID} --key-size 256 create ${ID_PART_ENTRY_UUID} /dev/${NAME} + bash -c 'while [ ! -e /dev/mapper/${ID_PART_ENTRY_UUID} ];do sleep 1; done' + /usr/sbin/ceph-disk-activate --mount /dev/mapper/${ID_PART_ENTRY_UUID} + ;; + +89c57f98-2fe5-4dc0-89c1-f3ad0ceff2be) + # TOBE_UUID + ;; + +89c57f98-2fe5-4dc0-89c1-5ec00ceff2be) + # DMCRYPT_TOBE_UUID + ;; + +*) + # Not a Ceph device + ;; + +esac + +exit diff --git a/udev/95-ceph-osd-alt.rules b/udev/95-ceph-osd-alt.rules new file mode 100644 index 0000000000000..702ceea5d39e4 --- /dev/null +++ b/udev/95-ceph-osd-alt.rules @@ -0,0 +1,5 @@ +# Check gpt partion for ceph tags and activate +ACTION=="add", SUBSYSTEM=="block", \ + ENV{DEVTYPE}=="partition", \ + ENV{ID_PART_TABLE_TYPE}=="gpt", \ + RUN+="/usr/sbin/ceph-disk-udev $number $name"