From 5c9ecea87d3eb90201a967d8da827018cd728f0d Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 18 May 2016 17:50:07 -0400 Subject: [PATCH 1/2] cls_rbd: async version of metadata_list helper method Signed-off-by: Jason Dillaman (cherry picked from commit 985cb38211c51c95d84479df231c4f53847cb2ec) --- src/cls/rbd/cls_rbd_client.cc | 34 +++++++++++++++++++++++++--------- src/cls/rbd/cls_rbd_client.h | 4 ++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/cls/rbd/cls_rbd_client.cc b/src/cls/rbd/cls_rbd_client.cc index ec57f7f8daae2..e7bd1cf05716b 100644 --- a/src/cls/rbd/cls_rbd_client.cc +++ b/src/cls/rbd/cls_rbd_client.cc @@ -942,21 +942,37 @@ namespace librbd { const std::string &start, uint64_t max_return, map *pairs) { - assert(pairs); - bufferlist in, out; - ::encode(start, in); - ::encode(max_return, in); - int r = ioctx->exec(oid, "rbd", "metadata_list", in, out); - if (r < 0) + librados::ObjectReadOperation op; + metadata_list_start(&op, start, max_return); + + bufferlist out_bl; + int r = ioctx->operate(oid, &op, &out_bl); + if (r < 0) { return r; + } - bufferlist::iterator iter = out.begin(); + bufferlist::iterator it = out_bl.begin(); + return metadata_list_finish(&it, pairs); + } + + void metadata_list_start(librados::ObjectReadOperation *op, + const std::string &start, uint64_t max_return) + { + bufferlist in_bl; + ::encode(start, in_bl); + ::encode(max_return, in_bl); + op->exec("rbd", "metadata_list", in_bl); + } + + int metadata_list_finish(bufferlist::iterator *it, + std::map *pairs) + { + assert(pairs); try { - ::decode(*pairs, iter); + ::decode(*pairs, *it); } catch (const buffer::error &err) { return -EBADMSG; } - return 0; } diff --git a/src/cls/rbd/cls_rbd_client.h b/src/cls/rbd/cls_rbd_client.h index b3dd22eb3753b..c23e143cd2867 100644 --- a/src/cls/rbd/cls_rbd_client.h +++ b/src/cls/rbd/cls_rbd_client.h @@ -135,6 +135,10 @@ namespace librbd { int metadata_list(librados::IoCtx *ioctx, const std::string &oid, const std::string &start, uint64_t max_return, map *pairs); + void metadata_list_start(librados::ObjectReadOperation *op, + const std::string &start, uint64_t max_return); + int metadata_list_finish(bufferlist::iterator *it, + std::map *pairs); int metadata_set(librados::IoCtx *ioctx, const std::string &oid, const map &data); int metadata_remove(librados::IoCtx *ioctx, const std::string &oid, From 305ebbc36473cf8bfd22513ffe55d3a4a99d5ef1 Mon Sep 17 00:00:00 2001 From: Jason Dillaman Date: Wed, 18 May 2016 19:19:24 -0400 Subject: [PATCH 2/2] librbd: metadata retrieval added to open image state machine Fixes: http://tracker.ceph.com/issues/15928 Signed-off-by: Jason Dillaman (cherry picked from commit b64cb31f8e7796175b4709929c017b3236649462) --- src/librbd/ImageCtx.cc | 66 ++++++++--------------- src/librbd/ImageCtx.h | 4 +- src/librbd/image/OpenRequest.cc | 93 ++++++++++++++++++++++++++++----- src/librbd/image/OpenRequest.h | 65 +++++++++++++---------- 4 files changed, 142 insertions(+), 86 deletions(-) diff --git a/src/librbd/ImageCtx.cc b/src/librbd/ImageCtx.cc index fa05103168664..564996985f9c6 100644 --- a/src/librbd/ImageCtx.cc +++ b/src/librbd/ImageCtx.cc @@ -234,10 +234,6 @@ struct C_InvalidateCache : public Context { void ImageCtx::init() { assert(!header_oid.empty()); assert(old_format || !id.empty()); - if (!old_format) { - init_layout(); - } - apply_metadata_confs(); asok_hook = new LibrbdAdminSocketHook(this); @@ -888,31 +884,32 @@ struct C_InvalidateCache : public Context { completed_reqs.clear(); } - bool ImageCtx::_filter_metadata_confs(const string &prefix, map &configs, - map &pairs, map *res) { + bool ImageCtx::_filter_metadata_confs(const string &prefix, + map &configs, + const map &pairs, + map *res) { size_t conf_prefix_len = prefix.size(); string start = prefix; - for (map::iterator it = pairs.begin(); it != pairs.end(); ++it) { - if (it->first.compare(0, MIN(conf_prefix_len, it->first.size()), prefix) > 0) + for (auto it : pairs) { + if (it.first.compare(0, MIN(conf_prefix_len, it.first.size()), prefix) > 0) return false; - if (it->first.size() <= conf_prefix_len) + if (it.first.size() <= conf_prefix_len) continue; - string key = it->first.substr(conf_prefix_len, it->first.size() - conf_prefix_len); - map::iterator cit = configs.find(key); - if ( cit != configs.end()) { + string key = it.first.substr(conf_prefix_len, it.first.size() - conf_prefix_len); + auto cit = configs.find(key); + if (cit != configs.end()) { cit->second = true; - res->insert(make_pair(key, it->second)); + res->insert(make_pair(key, it.second)); } } return true; } - void ImageCtx::apply_metadata_confs() { + void ImageCtx::apply_metadata(const std::map &meta) { ldout(cct, 20) << __func__ << dendl; - static uint64_t max_conf_items = 128; std::map configs = boost::assign::map_list_of( "rbd_non_blocking_aio", false)( "rbd_cache", false)( @@ -943,39 +940,18 @@ struct C_InvalidateCache : public Context { "rbd_journal_object_flush_age", false)( "rbd_journal_pool", false); - string start = METADATA_CONF_PREFIX; md_config_t local_config_t; - - bool retrieve_metadata = !old_format; - while (retrieve_metadata) { - map pairs, res; - int r = cls_client::metadata_list(&md_ctx, header_oid, start, max_conf_items, - &pairs); - if (r == -EOPNOTSUPP || r == -EIO) { - ldout(cct, 10) << "config metadata not supported by OSD" << dendl; - break; - } else if (r < 0) { - lderr(cct) << __func__ << " couldn't list config metadata: " << r + std::map res; + + _filter_metadata_confs(METADATA_CONF_PREFIX, configs, meta, &res); + for (auto it : res) { + std::string val(it.second.c_str(), it.second.length()); + int j = local_config_t.set_val(it.first.c_str(), val); + if (j < 0) { + lderr(cct) << __func__ << " failed to set config " << it.first + << " with value " << it.second.c_str() << ": " << j << dendl; - break; - } - if (pairs.empty()) { - break; - } - - retrieve_metadata = _filter_metadata_confs(METADATA_CONF_PREFIX, configs, - pairs, &res); - for (map::iterator it = res.begin(); - it != res.end(); ++it) { - string val(it->second.c_str(), it->second.length()); - int j = local_config_t.set_val(it->first.c_str(), val); - if (j < 0) { - lderr(cct) << __func__ << " failed to set config " << it->first - << " with value " << it->second.c_str() << ": " << j - << dendl; - } } - start = pairs.rbegin()->first; } #define ASSIGN_OPTION(config) \ diff --git a/src/librbd/ImageCtx.h b/src/librbd/ImageCtx.h index 3b58c6699ff76..076072c7a0187 100644 --- a/src/librbd/ImageCtx.h +++ b/src/librbd/ImageCtx.h @@ -192,7 +192,7 @@ namespace librbd { journal::Policy *journal_policy = nullptr; static bool _filter_metadata_confs(const string &prefix, std::map &configs, - map &pairs, map *res); + const map &pairs, map *res); // unit test mock helpers static ImageCtx* create(const std::string &image_name, @@ -285,7 +285,7 @@ namespace librbd { void cancel_async_requests(); void cancel_async_requests(Context *on_finish); - void apply_metadata_confs(); + void apply_metadata(const std::map &meta); ExclusiveLock *create_exclusive_lock(); ObjectMap *create_object_map(uint64_t snap_id); diff --git a/src/librbd/image/OpenRequest.cc b/src/librbd/image/OpenRequest.cc index 3027ecadab5f9..41f753e1c378b 100644 --- a/src/librbd/image/OpenRequest.cc +++ b/src/librbd/image/OpenRequest.cc @@ -11,6 +11,8 @@ #include "librbd/image/CloseRequest.h" #include "librbd/image/RefreshRequest.h" #include "librbd/image/SetSnapRequest.h" +#include +#include "include/assert.h" #define dout_subsys ceph_subsys_rbd #undef dout_prefix @@ -19,12 +21,19 @@ namespace librbd { namespace image { +namespace { + +static uint64_t MAX_METADATA_ITEMS = 128; + +} + using util::create_context_callback; using util::create_rados_ack_callback; template OpenRequest::OpenRequest(I *image_ctx, Context *on_finish) - : m_image_ctx(image_ctx), m_on_finish(on_finish), m_error_result(0) { + : m_image_ctx(image_ctx), m_on_finish(on_finish), m_error_result(0), + m_last_metadata_key(ImageCtx::METADATA_CONF_PREFIX) { } template @@ -63,6 +72,8 @@ Context *OpenRequest::handle_v1_detect_header(int *result) { m_image_ctx->old_format = true; m_image_ctx->header_oid = util::old_header_name(m_image_ctx->name); + m_image_ctx->apply_metadata({}); + send_register_watch(); } return nullptr; @@ -260,9 +271,66 @@ Context *OpenRequest::handle_v2_get_stripe_unit_count(int *result) { lderr(cct) << "failed to read striping metadata: " << cpp_strerror(*result) << dendl; send_close_image(*result); - } else { - send_register_watch(); + return nullptr; + } + + m_image_ctx->init_layout(); + + send_v2_apply_metadata(); + return nullptr; +} + +template +void OpenRequest::send_v2_apply_metadata() { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": " + << "start_key=" << m_last_metadata_key << dendl; + + librados::ObjectReadOperation op; + cls_client::metadata_list_start(&op, m_last_metadata_key, MAX_METADATA_ITEMS); + + using klass = OpenRequest; + librados::AioCompletion *comp = + create_rados_ack_callback(this); + m_out_bl.clear(); + m_image_ctx->md_ctx.aio_operate(m_image_ctx->header_oid, comp, &op, + &m_out_bl); + comp->release(); +} + +template +Context *OpenRequest::handle_v2_apply_metadata(int *result) { + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << ": r=" << *result << dendl; + + std::map metadata; + if (*result == 0) { + bufferlist::iterator it = m_out_bl.begin(); + *result = cls_client::metadata_list_finish(&it, &metadata); + } + + if (*result == -EOPNOTSUPP || *result == -EIO) { + ldout(cct, 10) << "config metadata not supported by OSD" << dendl; + } else if (*result < 0) { + lderr(cct) << "failed to retrieve metadata: " << cpp_strerror(*result) + << dendl; + send_close_image(*result); + return nullptr; } + + if (!metadata.empty()) { + m_metadata.insert(metadata.begin(), metadata.end()); + m_last_metadata_key = metadata.rbegin()->first; + if (boost::starts_with(m_last_metadata_key, + ImageCtx::METADATA_CONF_PREFIX)) { + send_v2_apply_metadata(); + return nullptr; + } + } + + m_image_ctx->apply_metadata(m_metadata); + + send_register_watch(); return nullptr; } @@ -270,17 +338,18 @@ template void OpenRequest::send_register_watch() { m_image_ctx->init(); - if (!m_image_ctx->read_only) { - CephContext *cct = m_image_ctx->cct; - ldout(cct, 10) << this << " " << __func__ << dendl; - - using klass = OpenRequest; - Context *ctx = create_context_callback< - klass, &klass::handle_register_watch>(this); - m_image_ctx->register_watch(ctx); - } else { + if (m_image_ctx->read_only) { send_refresh(); + return; } + + CephContext *cct = m_image_ctx->cct; + ldout(cct, 10) << this << " " << __func__ << dendl; + + using klass = OpenRequest; + Context *ctx = create_context_callback< + klass, &klass::handle_register_watch>(this); + m_image_ctx->register_watch(ctx); } template diff --git a/src/librbd/image/OpenRequest.h b/src/librbd/image/OpenRequest.h index 48aed3787ee7d..cf506ebe9743b 100644 --- a/src/librbd/image/OpenRequest.h +++ b/src/librbd/image/OpenRequest.h @@ -6,6 +6,8 @@ #include "include/int_types.h" #include "include/buffer.h" +#include +#include class Context; @@ -30,33 +32,36 @@ class OpenRequest { * * * | - * | (v1) (read only) - * |-----> V1_DETECT_HEADER . . . . . . . . . . . . . . . . . - * | | . - * | \-------------------------------\ . - * | (v2) | . - * \-----> V2_DETECT_HEADER | . - * | | . - * v | . - * V2_GET_ID|NAME | . - * | | . - * v | . - * V2_GET_IMMUTABLE_METADATA | . - * | | . - * v v . - * V2_GET_STRIPE_UNIT_COUNT ----> REGISTER_WATCH . - * . | . - * . (read only) v . - * . . . . . . . . . . . . . > REFRESH < . . . . . - * . | - * . | - * . \--> SET_SNAP - * (no snap) . | - * . v - * . . . > - * ^ - * (on error) | - * * * * * * * > CLOSE --------------------------------/ + * | (v1) + * |-----> V1_DETECT_HEADER + * | | + * | \-------------------------------\ + * | (v2) | + * \-----> V2_DETECT_HEADER | + * | | + * v | + * V2_GET_ID|NAME | + * | | + * v | + * V2_GET_IMMUTABLE_METADATA | + * | | + * v | + * V2_GET_STRIPE_UNIT_COUNT | + * | | + * v v + * /---> V2_APPLY_METADATA -------------> REGISTER_WATCH (skip if + * | | | read-only) + * \---------/ v + * REFRESH + * | + * v + * SET_SNAP (skip if no snap) + * | + * v + * + * ^ + * (on error) | + * * * * * * * > CLOSE ------------------------/ * * @endverbatim */ @@ -69,6 +74,9 @@ class OpenRequest { bufferlist m_out_bl; int m_error_result; + std::string m_last_metadata_key; + std::map m_metadata; + void send_v1_detect_header(); Context *handle_v1_detect_header(int *result); @@ -87,6 +95,9 @@ class OpenRequest { void send_v2_get_stripe_unit_count(); Context *handle_v2_get_stripe_unit_count(int *result); + void send_v2_apply_metadata(); + Context *handle_v2_apply_metadata(int *result); + void send_register_watch(); Context *handle_register_watch(int *result);