Merge pull request #14785 from idryomov/wip-krbd-exclusive-option

rbd: recognize exclusive option

Reviewed-by: Mykola Golub <mgolub@mirantis.com>

doc/man/8/rbd.rst

@@ -484,6 +484,8 @@ Per mapping (block device) `rbd map` options:
 * lock_on_read - Acquire exclusive lock on reads, in addition to writes and
   discards (since 4.9).
 
+* exclusive - Disable automatic exclusive lock transitions (since 4.12).
+
 `rbd unmap` options:
 
 * force - Force the unmapping of a block device that is open (since 4.9).  The

qa/suites/krbd/rbd-nomount/tasks/krbd_exclusive_option.yaml

@@ -0,0 +1,5 @@
+tasks:
+- workunit:
+    clients:
+      all:
+        - rbd/krbd_exclusive_option.sh

qa/workunits/rbd/krbd_exclusive_option.sh

@@ -0,0 +1,165 @@
+#!/bin/bash
+
+set -ex
+
+function expect_false() {
+    if "$@"; then return 1; else return 0; fi
+}
+
+function assert_locked() {
+    local dev_id="${1#/dev/rbd}"
+
+    local client_addr
+    client_addr="$(< $SYSFS_DIR/$dev_id/client_addr)"
+
+    local client_id
+    client_id="$(< $SYSFS_DIR/$dev_id/client_id)"
+    # client4324 -> client.4324
+    client_id="client.${client_id#client}"
+
+    local watch_cookie
+    watch_cookie="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID |
+        grep $client_id | cut -d ' ' -f 3 | cut -d '=' -f 2)"
+    [[ $(echo -n "$watch_cookie" | grep -c '^') -eq 1 ]]
+
+    local actual
+    actual="$(rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock |
+        python -m json.tool)"
+
+    local expected
+    expected="$(cat <<EOF | python -m json.tool
+{
+    "lockers": [
+        {
+            "addr": "$client_addr",
+            "cookie": "auto $watch_cookie",
+            "description": "",
+            "expiration": "0.000000",
+            "name": "$client_id"
+        }
+    ],
+    "name": "rbd_lock",
+    "tag": "internal",
+    "type": "exclusive"
+}
+EOF
+    )"
+
+    [ "$actual" = "$expected" ]
+}
+
+function assert_unlocked() {
+    rados -p rbd --format=json lock info rbd_header.$IMAGE_ID rbd_lock |
+        grep '"lockers":\[\]'
+}
+
+SYSFS_DIR="/sys/bus/rbd/devices"
+IMAGE_NAME="exclusive-option-test"
+
+rbd create --size 1 --image-feature '' $IMAGE_NAME
+
+IMAGE_ID="$(rbd info --format=json $IMAGE_NAME |
+    python -c "import sys, json; print json.load(sys.stdin)['block_name_prefix'].split('.')[1]")"
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_unlocked
+sudo rbd unmap $DEV
+assert_unlocked
+
+expect_false sudo rbd map -o exclusive $IMAGE_NAME
+assert_unlocked
+
+rbd feature enable $IMAGE_NAME exclusive-lock
+rbd snap create $IMAGE_NAME@snap
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_unlocked
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+[[ $(blockdev --getro $DEV) -eq 0 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME@snap)
+assert_locked $DEV
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive,ro $IMAGE_NAME)
+assert_locked $DEV
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+# alternate syntax
+DEV=$(sudo rbd map --exclusive --read-only $IMAGE_NAME)
+assert_locked $DEV
+[[ $(blockdev --getro $DEV) -eq 1 ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map $IMAGE_NAME)
+assert_unlocked
+dd if=/dev/urandom of=$DEV bs=4k count=10 oflag=direct
+assert_locked $DEV
+OTHER_DEV=$(sudo rbd map -o noshare,exclusive $IMAGE_NAME)
+assert_locked $OTHER_DEV
+sudo rbd unmap $DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+expect_false sudo rbd map -o noshare,exclusive $IMAGE_NAME
+assert_locked $DEV
+sudo rbd unmap $DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+OTHER_DEV=$(sudo rbd map -o noshare $IMAGE_NAME)
+dd if=/dev/urandom of=$OTHER_DEV bs=4k count=10 oflag=direct &
+PID=$!
+sleep 20
+assert_locked $DEV
+[ "$(ps -o stat= $PID)" = "D" ]
+sudo rbd unmap $DEV
+wait $PID
+assert_locked $OTHER_DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+
+DEV=$(sudo rbd map -o exclusive $IMAGE_NAME)
+assert_locked $DEV
+sudo rbd map -o noshare,lock_on_read $IMAGE_NAME &
+SUDO_PID=$!
+sleep 20
+assert_locked $DEV
+PID="$(ps -o pid= --ppid $SUDO_PID)"
+[ "$(ps -o stat= $PID)" = "Dl" ]
+sudo rbd unmap $DEV
+wait $SUDO_PID
+assert_locked $OTHER_DEV
+sudo rbd unmap $OTHER_DEV
+assert_unlocked
+
+# induce a watch error after 30 seconds
+DEV=$(sudo rbd map -o exclusive,osdkeepalive=60 $IMAGE_NAME)
+assert_locked $DEV
+OLD_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)"
+sleep 40
+assert_locked $DEV
+NEW_WATCHER="$(rados -p rbd listwatchers rbd_header.$IMAGE_ID)"
+# same client_id, old cookie < new cookie
+[ "$(echo "$OLD_WATCHER" | cut -d ' ' -f 2)" = \
+    "$(echo "$NEW_WATCHER" | cut -d ' ' -f 2)" ]
+[[ $(echo "$OLD_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) -lt \
+    $(echo "$NEW_WATCHER" | cut -d ' ' -f 3 | cut -d '=' -f 2) ]]
+sudo rbd unmap $DEV
+assert_unlocked
+
+echo OK

src/test/cli/rbd/help.t

@@ -874,7 +874,7 @@
 
   rbd help map
   usage: rbd map [--pool <pool>] [--image <image>] [--snap <snap>] 
-                 [--options <options>] [--read-only] 
+                 [--options <options>] [--read-only] [--exclusive] 
                  <image-or-snap-spec> 
 
   Map image to a block device using the kernel.
@@ -889,6 +889,7 @@
     --snap arg            snapshot name
     -o [ --options ] arg  map options
     --read-only           map read-only
+    --exclusive           disable automatic exclusive lock transitions
 
   rbd help merge-diff
   usage: rbd merge-diff [--path <path>] [--no-progress] 

src/tools/rbd/action/Kernel.cc

@@ -133,6 +133,8 @@ static int parse_map_options(char *options)
         return -EINVAL;
     } else if (!strcmp(this_char, "lock_on_read")) {
       put_map_option("lock_on_read", this_char);
+    } else if (!strcmp(this_char, "exclusive")) {
+      put_map_option("exclusive", this_char);
     } else {
       std::cerr << "rbd: unknown map option '" << this_char << "'" << std::endl;
       return -EINVAL;
@@ -385,7 +387,8 @@ void get_map_arguments(po::options_description *positional,
                                      at::ARGUMENT_MODIFIER_NONE);
   options->add_options()
     ("options,o", po::value<std::string>(), "map options")
-    ("read-only", po::bool_switch(), "map read-only");
+    ("read-only", po::bool_switch(), "map read-only")
+    ("exclusive", po::bool_switch(), "disable automatic exclusive lock transitions");
 }
 
 int execute_map(const po::variables_map &vm) {
@@ -404,6 +407,9 @@ int execute_map(const po::variables_map &vm) {
   if (vm["read-only"].as<bool>()) {
     put_map_option("rw", "ro");
   }
+  if (vm["exclusive"].as<bool>()) {
+    put_map_option("exclusive", "exclusive");
+  }
 
   // parse default options first so they can be overwritten by cli options
   char *default_map_options = strdup(g_conf->rbd_default_map_options.c_str());
@@ -503,7 +509,7 @@ int execute_unmap(const po::variables_map &vm) {
   return 0;
 }
 
-Shell::SwitchArguments switched_arguments({"read-only"});
+Shell::SwitchArguments switched_arguments({"read-only", "exclusive"});
 Shell::Action action_show(
   {"showmapped"}, {}, "Show the rbd images mapped by the kernel.", "",
   &get_show_arguments, &execute_show);