Skip to content

Commit

Permalink
Merge pull request #12839 from dillaman/wip-17993
Browse files Browse the repository at this point in the history 
librbd: delay mirror registration when creating clones

Reviewed-by: Mykola Golub <mgolub@mirantis.com>
  • Loading branch information
Mykola Golub committed Jan 11, 2017
2 parents 31f76d1 + 778e112 commit 3aca476
Show file tree
Hide file tree
Showing 12 changed files with 164 additions and 191 deletions.
37 changes: 25 additions & 12 deletions src/librbd/Journal.cc
View file
Expand Up @@ -53,25 +53,29 @@ class ThreadPoolSingleton : public ThreadPool {

template <typename I>
struct C_IsTagOwner : public Context {
I *image_ctx;
librados::IoCtx &io_ctx;
std::string image_id;
bool *is_tag_owner;
ContextWQ *op_work_queue;
Context *on_finish;

CephContext *cct = nullptr;
Journaler *journaler;
cls::journal::Client client;
journal::ImageClientMeta client_meta;
uint64_t tag_tid;
journal::TagData tag_data;

C_IsTagOwner(I *image_ctx, bool *is_tag_owner, Context *on_finish)
: image_ctx(image_ctx), is_tag_owner(is_tag_owner), on_finish(on_finish),
journaler(new Journaler(image_ctx->md_ctx, image_ctx->id,
Journal<>::IMAGE_CLIENT_ID, {})) {
C_IsTagOwner(librados::IoCtx &io_ctx, const std::string &image_id,
bool *is_tag_owner, ContextWQ *op_work_queue, Context *on_finish)
: io_ctx(io_ctx), image_id(image_id), is_tag_owner(is_tag_owner),
op_work_queue(op_work_queue), on_finish(on_finish),
cct(reinterpret_cast<CephContext*>(io_ctx.cct())),
journaler(new Journaler(io_ctx, image_id, Journal<>::IMAGE_CLIENT_ID,
{})) {
}

virtual void finish(int r) {
CephContext *cct = image_ctx->cct;

ldout(cct, 20) << this << " C_IsTagOwner::" << __func__ << ": r=" << r
<< dendl;
if (r < 0) {
Expand All @@ -88,7 +92,7 @@ struct C_IsTagOwner : public Context {
on_finish->complete(r);
delete journaler;
});
image_ctx->op_work_queue->queue(ctx, r);
op_work_queue->queue(ctx, r);
}
};

Expand Down Expand Up @@ -432,7 +436,8 @@ int Journal<I>::reset(librados::IoCtx &io_ctx, const std::string &image_id) {

template <typename I>
int Journal<I>::is_tag_owner(I *image_ctx, bool *is_tag_owner) {
return Journal<>::is_tag_owner(image_ctx->md_ctx, image_ctx->id, is_tag_owner);
return Journal<>::is_tag_owner(image_ctx->md_ctx, image_ctx->id,
is_tag_owner);
}

template <typename I>
Expand All @@ -449,13 +454,21 @@ int Journal<I>::is_tag_owner(IoCtx& io_ctx, std::string& image_id,
}

template <typename I>
void Journal<I>::is_tag_owner(I *image_ctx, bool *is_tag_owner,
void Journal<I>::is_tag_owner(I *image_ctx, bool *owner,
Context *on_finish) {
CephContext *cct = image_ctx->cct;
is_tag_owner(image_ctx->md_ctx, image_ctx->id, owner,
image_ctx->op_work_queue, on_finish);
}

template <typename I>
void Journal<I>::is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id,
bool *is_tag_owner, ContextWQ *op_work_queue,
Context *on_finish) {
CephContext *cct = reinterpret_cast<CephContext*>(io_ctx.cct());
ldout(cct, 20) << __func__ << dendl;

C_IsTagOwner<I> *is_tag_owner_ctx = new C_IsTagOwner<I>(
image_ctx, is_tag_owner, on_finish);
io_ctx, image_id, is_tag_owner, op_work_queue, on_finish);
get_tags(cct, is_tag_owner_ctx->journaler, &is_tag_owner_ctx->client,
&is_tag_owner_ctx->client_meta, &is_tag_owner_ctx->tag_tid,
&is_tag_owner_ctx->tag_data, is_tag_owner_ctx);
Expand Down
3 changes: 3 additions & 0 deletions src/librbd/Journal.h
View file
Expand Up @@ -107,6 +107,9 @@ class Journal {
bool *is_tag_owner);
static void is_tag_owner(ImageCtxT *image_ctx, bool *is_tag_owner,
Context *on_finish);
static void is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id,
bool *is_tag_owner, ContextWQ *op_work_queue,
Context *on_finish);
static int get_tag_owner(ImageCtxT *image_ctx, std::string *mirror_uuid);
static int get_tag_owner(librados::IoCtx& io_ctx, std::string& image_id,
std::string *mirror_uuid);
Expand Down
125 changes: 17 additions & 108 deletions src/librbd/image/CreateRequest.cc
View file
Expand Up @@ -10,10 +10,11 @@
#include "common/ceph_context.h"
#include "librbd/AioCompletion.h"
#include "librbd/Journal.h"
#include "librbd/MirroringWatcher.h"
#include "librbd/journal/CreateRequest.h"
#include "librbd/journal/RemoveRequest.h"
#include "librbd/mirror/EnableRequest.h"
#include "journal/Journaler.h"
#include "librbd/MirroringWatcher.h"

#define dout_subsys ceph_subsys_rbd
#undef dout_prefix
Expand Down Expand Up @@ -122,10 +123,12 @@ CreateRequest<I>::CreateRequest(IoCtx &ioctx, const std::string &image_name,
const ImageOptions &image_options,
const std::string &non_primary_global_image_id,
const std::string &primary_mirror_uuid,
bool skip_mirror_enable,
ContextWQ *op_work_queue, Context *on_finish)
: m_image_name(image_name), m_image_id(image_id), m_size(size),
m_non_primary_global_image_id(non_primary_global_image_id),
m_primary_mirror_uuid(primary_mirror_uuid),
m_skip_mirror_enable(skip_mirror_enable),
m_op_work_queue(op_work_queue), m_on_finish(on_finish) {
m_ioctx.dup(ioctx);
m_cct = reinterpret_cast<CephContext *>(m_ioctx.cct());
Expand Down Expand Up @@ -639,134 +642,40 @@ Context* CreateRequest<I>::handle_journal_create(int *result) {
return nullptr;
}

fetch_mirror_image();
mirror_image_enable();
return nullptr;
}

template<typename I>
void CreateRequest<I>::fetch_mirror_image() {
if ((m_mirror_mode != RBD_MIRROR_MODE_POOL) && !m_force_non_primary) {
void CreateRequest<I>::mirror_image_enable() {
if (((m_mirror_mode != RBD_MIRROR_MODE_POOL) && !m_force_non_primary) ||
m_skip_mirror_enable) {
complete(0);
return;
}

ldout(m_cct, 20) << this << " " << __func__ << dendl;

librados::ObjectReadOperation op;
cls_client::mirror_image_get_start(&op, m_image_id);

using klass = CreateRequest<I>;
librados::AioCompletion *comp =
create_rados_ack_callback<klass, &klass::handle_fetch_mirror_image>(this);
m_outbl.clear();
int r = m_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_outbl);
assert(r == 0);
comp->release();
}

template<typename I>
Context *CreateRequest<I>::handle_fetch_mirror_image(int *result) {
ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl;

if ((*result < 0) && (*result != -ENOENT)) {
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result) << dendl;

m_r_saved = *result;
journal_remove();
return nullptr;
}

if (*result == 0) {
bufferlist::iterator it = m_outbl.begin();
*result = cls_client::mirror_image_get_finish(&it, &m_mirror_image_internal);
if (*result < 0) {
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result) << dendl;

m_r_saved = *result;
journal_remove();
return nullptr;
}

if (m_mirror_image_internal.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
return m_on_finish;
}
}

// enable image mirroring (-ENOENT or disabled earlier)
mirror_image_enable();
return nullptr;
}

template<typename I>
void CreateRequest<I>::mirror_image_enable() {
ldout(m_cct, 20) << this << " " << __func__ << dendl;

m_mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_ENABLED;
if (m_non_primary_global_image_id.empty()) {
uuid_d uuid_gen;
uuid_gen.generate_random();
m_mirror_image_internal.global_image_id = uuid_gen.to_string();
} else {
m_mirror_image_internal.global_image_id = m_non_primary_global_image_id;
}

librados::ObjectWriteOperation op;
cls_client::mirror_image_set(&op, m_image_id, m_mirror_image_internal);

using klass = CreateRequest<I>;
librados::AioCompletion *comp =
create_rados_ack_callback<klass, &klass::handle_mirror_image_enable>(this);
int r = m_ioctx.aio_operate(RBD_MIRRORING, comp, &op);
assert(r == 0);
comp->release();
auto ctx = create_context_callback<
CreateRequest<I>, &CreateRequest<I>::handle_mirror_image_enable>(this);
auto req = mirror::EnableRequest<I>::create(m_ioctx, m_image_id,
m_non_primary_global_image_id,
m_op_work_queue, ctx);
req->send();
}

template<typename I>
Context *CreateRequest<I>::handle_mirror_image_enable(int *result) {
ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl;

if (*result < 0) {
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result) << dendl;
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result)
<< dendl;

m_r_saved = *result;
journal_remove();
return nullptr;
}

send_watcher_notification();
return nullptr;
}

// TODO: make this *really* async
template<typename I>
void CreateRequest<I>::send_watcher_notification() {
ldout(m_cct, 20) << this << " " << __func__ << dendl;

Context *ctx = new FunctionContext([this](int r) {
r = MirroringWatcher<>::notify_image_updated(
m_ioctx, cls::rbd::MIRROR_IMAGE_STATE_ENABLED,
m_image_id, m_mirror_image_internal.global_image_id);
handle_watcher_notify(r);
});

m_op_work_queue->queue(ctx, 0);
}

template<typename I>
void CreateRequest<I>::handle_watcher_notify(int r) {
ldout(m_cct, 20) << __func__ << ": r=" << r << dendl;

if (r < 0) {
// non fatal error -- watchers would cope up upon noticing missing
// updates. just log and move on...
ldout(m_cct, 10) << "failed to send update notification: " << cpp_strerror(r)
<< dendl;
} else {
ldout(m_cct, 20) << "image mirroring is enabled: global_id=" <<
m_mirror_image_internal.global_image_id << dendl;
}

complete(0);
return m_on_finish;
}

template<typename I>
Expand Down
21 changes: 7 additions & 14 deletions src/librbd/image/CreateRequest.h
View file
Expand Up @@ -35,10 +35,12 @@ class CreateRequest {
const ImageOptions &image_options,
const std::string &non_primary_global_image_id,
const std::string &primary_mirror_uuid,
bool skip_mirror_enable,
ContextWQ *op_work_queue, Context *on_finish) {
return new CreateRequest(ioctx, image_name, image_id, size, image_options,
non_primary_global_image_id, primary_mirror_uuid,
op_work_queue, on_finish);
skip_mirror_enable, op_work_queue,
on_finish);
}

static int validate_order(CephContext *cct, uint8_t order);
Expand Down Expand Up @@ -79,14 +81,9 @@ class CreateRequest {
* | |\ JOURNAL CREATE .
* | | \ / | .
* v | *<------------/ v .
* | | FETCH MIRROR IMAGE v
* | | MIRROR IMAGE ENABLE .
* | | / | .
* | JOURNAL REMOVE<--------/ v .
* | \ MIRROR IMAGE ENABLE .
* | \ / | .
* | *<------------/ v .
* | NOTIFY WATCHERS .
* | | .
* | JOURNAL REMOVE*<-------/ | .
* | v .
* |_____________>___________________<finish> . . . . < . . . .
*
Expand All @@ -98,6 +95,7 @@ class CreateRequest {
const ImageOptions &image_options,
const std::string &non_primary_global_image_id,
const std::string &primary_mirror_uuid,
bool skip_mirror_enable,
ContextWQ *op_work_queue, Context *on_finish);

IoCtx m_ioctx;
Expand All @@ -115,6 +113,7 @@ class CreateRequest {
int64_t m_data_pool_id = -1;
const std::string m_non_primary_global_image_id;
const std::string m_primary_mirror_uuid;
bool m_skip_mirror_enable;
bool m_negotiate_features = false;

ContextWQ *m_op_work_queue;
Expand Down Expand Up @@ -157,15 +156,9 @@ class CreateRequest {
void journal_create();
Context *handle_journal_create(int *result);

void fetch_mirror_image();
Context *handle_fetch_mirror_image(int *result);

void mirror_image_enable();
Context *handle_mirror_image_enable(int *result);

void send_watcher_notification();
void handle_watcher_notify(int r);

void complete(int r);

// cleanup
Expand Down

0 comments on commit 3aca476

Please sign in to comment.