rbd: show info about mirror daemon instance in image mirror status output

qa/workunits/rbd/rbd_mirror_ha.sh

@@ -71,7 +71,8 @@ test_replay()
 	wait_for_replay_complete ${CLUSTER1}:${LEADER} ${CLUSTER2} ${POOL} \
 				 ${image}
 	wait_for_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+replaying' \
-				    'master_position'
+                                    'master_position' \
+                                    "${MIRROR_USER_ID_PREFIX}${LEADER} on $(hostname -s)"
 	if [ -z "${RBD_MIRROR_USE_RBD_MIRROR}" ]; then
 	    wait_for_status_in_pool_dir ${CLUSTER2} ${POOL} ${image} \
 					'down+unknown'

qa/workunits/rbd/rbd_mirror_helpers.sh

@@ -462,6 +462,8 @@ status()
     do
 	echo "${cluster} status"
 	ceph --cluster ${cluster} -s
+	ceph --cluster ${cluster} service dump
+	ceph --cluster ${cluster} service status
 	echo
 
 	for image_pool in ${POOL} ${PARENT_POOL}
@@ -673,27 +675,41 @@ test_status_in_pool_dir()
     local cluster=$1
     local pool=$2
     local image=$3
-    local state_pattern=$4
-    local description_pattern=$5
+    local state_pattern="$4"
+    local description_pattern="$5"
+    local service_pattern="$6"
 
     local status_log=${TEMPDIR}/${cluster}-${image}.mirror_status
     rbd --cluster ${cluster} -p ${pool} mirror image status ${image} |
 	tee ${status_log} >&2
     grep "state: .*${state_pattern}" ${status_log} || return 1
     grep "description: .*${description_pattern}" ${status_log} || return 1
+
+    if [ -n "${service_pattern}" ]; then
+        grep "service: *${service_pattern}" ${status_log} || return 1
+    elif echo ${state_pattern} | grep '^up+'; then
+        grep "service: *${MIRROR_USER_ID_PREFIX}.* on " ${status_log} || return 1
+    else
+        grep "service: " ${status_log} && return 1
+    fi
+
+    return 0
 }
 
 wait_for_status_in_pool_dir()
 {
     local cluster=$1
     local pool=$2
     local image=$3
-    local state_pattern=$4
-    local description_pattern=$5
+    local state_pattern="$4"
+    local description_pattern="$5"
+    local service_pattern="$6"
 
     for s in 1 2 4 8 8 8 8 8 8 8 8 16 16; do
 	sleep ${s}
-	test_status_in_pool_dir ${cluster} ${pool} ${image} ${state_pattern} ${description_pattern} && return 0
+	test_status_in_pool_dir ${cluster} ${pool} ${image} "${state_pattern}" \
+                                "${description_pattern}" "${service_pattern}" &&
+            return 0
     done
     return 1
 }

src/cls/rbd/cls_rbd.cc

@@ -4993,6 +4993,93 @@ int image_status_remove_down(cls_method_context_t hctx) {
   return 0;
 }
 
+int image_instance_get(cls_method_context_t hctx,
+                       const string &global_image_id,
+                       const std::set<entity_inst_t> &watchers,
+                       entity_inst_t *instance) {
+  bufferlist bl;
+  int r = cls_cxx_map_get_val(hctx, status_global_key(global_image_id), &bl);
+  if (r < 0) {
+    if (r != -ENOENT) {
+      CLS_ERR("error reading status for mirrored image, global id '%s': '%s'",
+              global_image_id.c_str(), cpp_strerror(r).c_str());
+    }
+    return r;
+  }
+
+  MirrorImageStatusOnDisk ondisk_status;
+  try {
+    auto it = bl.cbegin();
+    decode(ondisk_status, it);
+  } catch (const buffer::error &err) {
+    CLS_ERR("could not decode status for mirrored image, global id '%s'",
+            global_image_id.c_str());
+    return -EIO;
+  }
+
+  if (watchers.find(ondisk_status.origin) == watchers.end()) {
+    return -ESTALE;
+  }
+
+  *instance = ondisk_status.origin;
+  return 0;
+}
+
+int image_instance_list(cls_method_context_t hctx,
+                        const std::string &start_after,
+                        uint64_t max_return,
+                        map<std::string, entity_inst_t> *instances) {
+  std::string last_read = image_key(start_after);
+  int max_read = RBD_MAX_KEYS_READ;
+  bool more = true;
+
+  std::set<entity_inst_t> watchers;
+  int r = list_watchers(hctx, &watchers);
+  if (r < 0) {
+    return r;
+  }
+
+  while (more && instances->size() < max_return) {
+    std::map<std::string, bufferlist> vals;
+    CLS_LOG(20, "last_read = '%s'", last_read.c_str());
+    r = cls_cxx_map_get_vals(hctx, last_read, IMAGE_KEY_PREFIX, max_read, &vals,
+                             &more);
+    if (r < 0) {
+      CLS_ERR("error reading mirror image directory by name: %s",
+              cpp_strerror(r).c_str());
+      return r;
+    }
+
+    for (auto it = vals.begin(); it != vals.end() &&
+           instances->size() < max_return; ++it) {
+      const std::string &image_id = it->first.substr(IMAGE_KEY_PREFIX.size());
+      cls::rbd::MirrorImage mirror_image;
+      auto iter = it->second.cbegin();
+      try {
+        decode(mirror_image, iter);
+      } catch (const buffer::error &err) {
+        CLS_ERR("could not decode mirror image payload of image '%s'",
+                image_id.c_str());
+        return -EIO;
+      }
+
+      entity_inst_t instance;
+      r = image_instance_get(hctx, mirror_image.global_image_id, watchers,
+                             &instance);
+      if (r < 0) {
+        continue;
+      }
+
+      (*instances)[image_id] = instance;
+    }
+    if (!vals.empty()) {
+      last_read = vals.rbegin()->first;
+    }
+  }
+
+  return 0;
+}
+
 int instances_list(cls_method_context_t hctx,
                    std::vector<std::string> *instance_ids) {
   std::string last_read = INSTANCE_KEY_PREFIX;
@@ -5766,6 +5853,73 @@ int mirror_image_status_remove_down(cls_method_context_t hctx, bufferlist *in,
   return 0;
 }
 
+/**
+ * Input:
+ * @param global_image_id (std::string)
+ *
+ * Output:
+ * @param entity_inst_t - instance
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_image_instance_get(cls_method_context_t hctx, bufferlist *in,
+                              bufferlist *out) {
+  string global_image_id;
+  try {
+    auto it = in->cbegin();
+    decode(global_image_id, it);
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  std::set<entity_inst_t> watchers;
+  int r = mirror::list_watchers(hctx, &watchers);
+  if (r < 0) {
+    return r;
+  }
+
+  entity_inst_t instance;
+  r = mirror::image_instance_get(hctx, global_image_id, watchers, &instance);
+  if (r < 0) {
+    return r;
+  }
+
+  encode(instance, *out, cls_get_features(hctx));
+  return 0;
+}
+
+/**
+ * Input:
+ * @param start_after which name to begin listing after
+ *        (use the empty string to start at the beginning)
+ * @param max_return the maximum number of names to list
+ *
+ * Output:
+ * @param std::map<std::string, entity_inst_t>: image id to instance map
+ * @returns 0 on success, negative error code on failure
+ */
+int mirror_image_instance_list(cls_method_context_t hctx, bufferlist *in,
+                               bufferlist *out) {
+  std::string start_after;
+  uint64_t max_return;
+  try {
+    auto iter = in->cbegin();
+    decode(start_after, iter);
+    decode(max_return, iter);
+  } catch (const buffer::error &err) {
+    return -EINVAL;
+  }
+
+  map<std::string, entity_inst_t> instances;
+  int r = mirror::image_instance_list(hctx, start_after, max_return,
+                                      &instances);
+  if (r < 0) {
+    return r;
+  }
+
+  encode(instances, *out, cls_get_features(hctx));
+  return 0;
+}
+
 /**
  * Input:
  * none
@@ -7190,6 +7344,8 @@ CLS_INIT(rbd)
   cls_method_handle_t h_mirror_image_status_list;
   cls_method_handle_t h_mirror_image_status_get_summary;
   cls_method_handle_t h_mirror_image_status_remove_down;
+  cls_method_handle_t h_mirror_image_instance_get;
+  cls_method_handle_t h_mirror_image_instance_list;
   cls_method_handle_t h_mirror_instances_list;
   cls_method_handle_t h_mirror_instances_add;
   cls_method_handle_t h_mirror_instances_remove;
@@ -7513,6 +7669,12 @@ CLS_INIT(rbd)
                           CLS_METHOD_RD | CLS_METHOD_WR,
                           mirror_image_status_remove_down,
 			  &h_mirror_image_status_remove_down);
+  cls_register_cxx_method(h_class, "mirror_image_instance_get", CLS_METHOD_RD,
+                          mirror_image_instance_get,
+                          &h_mirror_image_instance_get);
+  cls_register_cxx_method(h_class, "mirror_image_instance_list", CLS_METHOD_RD,
+                          mirror_image_instance_list,
+                          &h_mirror_image_instance_list);
   cls_register_cxx_method(h_class, "mirror_instances_list", CLS_METHOD_RD,
                           mirror_instances_list, &h_mirror_instances_list);
   cls_register_cxx_method(h_class, "mirror_instances_add",

src/cls/rbd/cls_rbd_client.cc

@@ -2170,6 +2170,84 @@ void mirror_image_status_remove_down(librados::ObjectWriteOperation *op) {
   op->exec("rbd", "mirror_image_status_remove_down", bl);
 }
 
+int mirror_image_instance_get(librados::IoCtx *ioctx,
+                              const std::string &global_image_id,
+                              entity_inst_t *instance) {
+  librados::ObjectReadOperation op;
+  mirror_image_instance_get_start(&op, global_image_id);
+
+  bufferlist out_bl;
+  int r = ioctx->operate(RBD_MIRRORING, &op, &out_bl);
+  if (r < 0) {
+    return r;
+  }
+
+  auto iter = out_bl.cbegin();
+  r = mirror_image_instance_get_finish(&iter, instance);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+void mirror_image_instance_get_start(librados::ObjectReadOperation *op,
+                                     const std::string &global_image_id) {
+  bufferlist bl;
+  encode(global_image_id, bl);
+  op->exec("rbd", "mirror_image_instance_get", bl);
+}
+
+int mirror_image_instance_get_finish(bufferlist::const_iterator *iter,
+                                     entity_inst_t *instance) {
+  try {
+    decode(*instance, *iter);
+  } catch (const buffer::error &err) {
+    return -EBADMSG;
+  }
+  return 0;
+}
+
+int mirror_image_instance_list(
+    librados::IoCtx *ioctx, const std::string &start, uint64_t max_return,
+    std::map<std::string, entity_inst_t> *instances) {
+  librados::ObjectReadOperation op;
+  mirror_image_instance_list_start(&op, start, max_return);
+
+  bufferlist out_bl;
+  int r = ioctx->operate(RBD_MIRRORING, &op, &out_bl);
+  if (r < 0) {
+    return r;
+  }
+
+  auto iter = out_bl.cbegin();
+  r = mirror_image_instance_list_finish(&iter, instances);
+  if (r < 0) {
+    return r;
+  }
+  return 0;
+}
+
+void mirror_image_instance_list_start(librados::ObjectReadOperation *op,
+                                      const std::string &start,
+                                      uint64_t max_return) {
+  bufferlist bl;
+  encode(start, bl);
+  encode(max_return, bl);
+  op->exec("rbd", "mirror_image_instance_list", bl);
+}
+
+int mirror_image_instance_list_finish(
+    bufferlist::const_iterator *iter,
+    std::map<std::string, entity_inst_t> *instances) {
+  instances->clear();
+  try {
+    decode(*instances, *iter);
+  } catch (const buffer::error &err) {
+    return -EBADMSG;
+  }
+  return 0;
+}
+
 void mirror_instances_list_start(librados::ObjectReadOperation *op) {
   bufferlist bl;
   op->exec("rbd", "mirror_instances_list", bl);

src/cls/rbd/cls_rbd_client.h

@@ -474,6 +474,22 @@ int mirror_image_status_get_summary_finish(bufferlist::const_iterator *iter,
 int mirror_image_status_remove_down(librados::IoCtx *ioctx);
 void mirror_image_status_remove_down(librados::ObjectWriteOperation *op);
 
+int mirror_image_instance_get(librados::IoCtx *ioctx,
+                              const std::string &global_image_id,
+                              entity_inst_t *instance);
+void mirror_image_instance_get_start(librados::ObjectReadOperation *op,
+                                     const std::string &global_image_id);
+int mirror_image_instance_get_finish(bufferlist::const_iterator *iter,
+                                     entity_inst_t *instance);
+int mirror_image_instance_list(librados::IoCtx *ioctx,
+                               const std::string &start, uint64_t max_return,
+                               std::map<std::string, entity_inst_t> *instances);
+void mirror_image_instance_list_start(librados::ObjectReadOperation *op,
+                                      const std::string &start,
+                                      uint64_t max_return);
+int mirror_image_instance_list_finish(bufferlist::const_iterator *iter,
+                                      std::map<std::string, entity_inst_t> *instances);
+
 void mirror_instances_list_start(librados::ObjectReadOperation *op);
 int mirror_instances_list_finish(bufferlist::const_iterator *iter,
                                  std::vector<std::string> *instance_ids);

src/include/rbd/librbd.h

@@ -405,6 +405,15 @@ CEPH_RBD_API void rbd_mirror_image_status_list_cleanup(char **image_ids,
 CEPH_RBD_API int rbd_mirror_image_status_summary(rados_ioctx_t io_ctx,
     rbd_mirror_image_status_state_t *states, int *counts, size_t *maxlen);
 
+CEPH_RBD_API int rbd_mirror_image_instance_id_list(rados_ioctx_t io_ctx,
+                                                   const char *start_id,
+                                                   size_t max, char **image_ids,
+                                                   char **instance_ids,
+                                                   size_t *len);
+CEPH_RBD_API void rbd_mirror_image_instance_id_list_cleanup(char **image_ids,
+                                                            char **instance_ids,
+                                                            size_t len);
+
 /* pool metadata */
 CEPH_RBD_API int rbd_pool_metadata_get(rados_ioctx_t io_ctx, const char *key,
                                        char *value, size_t *val_len);
@@ -964,6 +973,9 @@ CEPH_RBD_API int rbd_mirror_image_get_info(rbd_image_t image,
 CEPH_RBD_API int rbd_mirror_image_get_status(rbd_image_t image,
                                              rbd_mirror_image_status_t *mirror_image_status,
                                              size_t status_size);
+CEPH_RBD_API int rbd_mirror_image_get_instance_id(rbd_image_t image,
+                                                  char *instance_id,
+                                                  size_t *id_max_length);
 CEPH_RBD_API int rbd_aio_mirror_image_promote(rbd_image_t image, bool force,
                                               rbd_completion_t c);
 CEPH_RBD_API int rbd_aio_mirror_image_demote(rbd_image_t image,