Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

librbd: delay mirror registration when creating clones #12839

Merged
merged 1 commit into from Jan 11, 2017
Merged

librbd: delay mirror registration when creating clones #12839

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
37 changes: 25 additions & 12 deletions src/librbd/Journal.cc
View file
Open in desktop
Expand Up @@ -53,25 +53,29 @@ class ThreadPoolSingleton : public ThreadPool {

template <typename I>
struct C_IsTagOwner : public Context {
I *image_ctx;
librados::IoCtx &io_ctx;
std::string image_id;
bool *is_tag_owner;
ContextWQ *op_work_queue;
Context *on_finish;

CephContext *cct = nullptr;
Journaler *journaler;
cls::journal::Client client;
journal::ImageClientMeta client_meta;
uint64_t tag_tid;
journal::TagData tag_data;

C_IsTagOwner(I *image_ctx, bool *is_tag_owner, Context *on_finish)
: image_ctx(image_ctx), is_tag_owner(is_tag_owner), on_finish(on_finish),
journaler(new Journaler(image_ctx->md_ctx, image_ctx->id,
Journal<>::IMAGE_CLIENT_ID, {})) {
C_IsTagOwner(librados::IoCtx &io_ctx, const std::string &image_id,
bool *is_tag_owner, ContextWQ *op_work_queue, Context *on_finish)
: io_ctx(io_ctx), image_id(image_id), is_tag_owner(is_tag_owner),
op_work_queue(op_work_queue), on_finish(on_finish),
cct(reinterpret_cast<CephContext*>(io_ctx.cct())),
journaler(new Journaler(io_ctx, image_id, Journal<>::IMAGE_CLIENT_ID,
{})) {
}

virtual void finish(int r) {
CephContext *cct = image_ctx->cct;

ldout(cct, 20) << this << " C_IsTagOwner::" << __func__ << ": r=" << r
<< dendl;
if (r < 0) {
Expand All @@ -88,7 +92,7 @@ struct C_IsTagOwner : public Context {
on_finish->complete(r);
delete journaler;
});
image_ctx->op_work_queue->queue(ctx, r);
op_work_queue->queue(ctx, r);
}
};

Expand Down Expand Up @@ -432,7 +436,8 @@ int Journal<I>::reset(librados::IoCtx &io_ctx, const std::string &image_id) {

template <typename I>
int Journal<I>::is_tag_owner(I *image_ctx, bool *is_tag_owner) {
return Journal<>::is_tag_owner(image_ctx->md_ctx, image_ctx->id, is_tag_owner);
return Journal<>::is_tag_owner(image_ctx->md_ctx, image_ctx->id,
is_tag_owner);
}

template <typename I>
Expand All @@ -449,13 +454,21 @@ int Journal<I>::is_tag_owner(IoCtx& io_ctx, std::string& image_id,
}

template <typename I>
void Journal<I>::is_tag_owner(I *image_ctx, bool *is_tag_owner,
void Journal<I>::is_tag_owner(I *image_ctx, bool *owner,
Context *on_finish) {
CephContext *cct = image_ctx->cct;
is_tag_owner(image_ctx->md_ctx, image_ctx->id, owner,
image_ctx->op_work_queue, on_finish);
}

template <typename I>
void Journal<I>::is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id,
bool *is_tag_owner, ContextWQ *op_work_queue,
Context *on_finish) {
CephContext *cct = reinterpret_cast<CephContext*>(io_ctx.cct());
ldout(cct, 20) << __func__ << dendl;

C_IsTagOwner<I> *is_tag_owner_ctx = new C_IsTagOwner<I>(
image_ctx, is_tag_owner, on_finish);
io_ctx, image_id, is_tag_owner, op_work_queue, on_finish);
get_tags(cct, is_tag_owner_ctx->journaler, &is_tag_owner_ctx->client,
&is_tag_owner_ctx->client_meta, &is_tag_owner_ctx->tag_tid,
&is_tag_owner_ctx->tag_data, is_tag_owner_ctx);
Expand Down
3 changes: 3 additions & 0 deletions src/librbd/Journal.h
View file
Open in desktop
Expand Up @@ -107,6 +107,9 @@ class Journal {
bool *is_tag_owner);
static void is_tag_owner(ImageCtxT *image_ctx, bool *is_tag_owner,
Context *on_finish);
static void is_tag_owner(librados::IoCtx& io_ctx, std::string& image_id,
bool *is_tag_owner, ContextWQ *op_work_queue,
Context *on_finish);
static int get_tag_owner(ImageCtxT *image_ctx, std::string *mirror_uuid);
static int get_tag_owner(librados::IoCtx& io_ctx, std::string& image_id,
std::string *mirror_uuid);
Expand Down
125 changes: 17 additions & 108 deletions src/librbd/image/CreateRequest.cc
View file
Open in desktop
Expand Up @@ -10,10 +10,11 @@
#include "common/ceph_context.h"
#include "librbd/AioCompletion.h"
#include "librbd/Journal.h"
#include "librbd/MirroringWatcher.h"
#include "librbd/journal/CreateRequest.h"
#include "librbd/journal/RemoveRequest.h"
#include "librbd/mirror/EnableRequest.h"
#include "journal/Journaler.h"
#include "librbd/MirroringWatcher.h"

#define dout_subsys ceph_subsys_rbd
#undef dout_prefix
Expand Down Expand Up @@ -122,10 +123,12 @@ CreateRequest<I>::CreateRequest(IoCtx &ioctx, const std::string &image_name,
const ImageOptions &image_options,
const std::string &non_primary_global_image_id,
const std::string &primary_mirror_uuid,
bool skip_mirror_enable,
ContextWQ *op_work_queue, Context *on_finish)
: m_image_name(image_name), m_image_id(image_id), m_size(size),
m_non_primary_global_image_id(non_primary_global_image_id),
m_primary_mirror_uuid(primary_mirror_uuid),
m_skip_mirror_enable(skip_mirror_enable),
m_op_work_queue(op_work_queue), m_on_finish(on_finish) {
m_ioctx.dup(ioctx);
m_cct = reinterpret_cast<CephContext *>(m_ioctx.cct());
Expand Down Expand Up @@ -639,134 +642,40 @@ Context* CreateRequest<I>::handle_journal_create(int *result) {
return nullptr;
}

fetch_mirror_image();
mirror_image_enable();
return nullptr;
}

template<typename I>
void CreateRequest<I>::fetch_mirror_image() {
if ((m_mirror_mode != RBD_MIRROR_MODE_POOL) && !m_force_non_primary) {
void CreateRequest<I>::mirror_image_enable() {
if (((m_mirror_mode != RBD_MIRROR_MODE_POOL) && !m_force_non_primary) ||
m_skip_mirror_enable) {
complete(0);
return;
}

ldout(m_cct, 20) << this << " " << __func__ << dendl;

librados::ObjectReadOperation op;
cls_client::mirror_image_get_start(&op, m_image_id);

using klass = CreateRequest<I>;
librados::AioCompletion *comp =
create_rados_ack_callback<klass, &klass::handle_fetch_mirror_image>(this);
m_outbl.clear();
int r = m_ioctx.aio_operate(RBD_MIRRORING, comp, &op, &m_outbl);
assert(r == 0);
comp->release();
}

template<typename I>
Context *CreateRequest<I>::handle_fetch_mirror_image(int *result) {
ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl;

if ((*result < 0) && (*result != -ENOENT)) {
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result) << dendl;

m_r_saved = *result;
journal_remove();
return nullptr;
}

if (*result == 0) {
bufferlist::iterator it = m_outbl.begin();
*result = cls_client::mirror_image_get_finish(&it, &m_mirror_image_internal);
if (*result < 0) {
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result) << dendl;

m_r_saved = *result;
journal_remove();
return nullptr;
}

if (m_mirror_image_internal.state == cls::rbd::MIRROR_IMAGE_STATE_ENABLED) {
return m_on_finish;
}
}

// enable image mirroring (-ENOENT or disabled earlier)
mirror_image_enable();
return nullptr;
}

template<typename I>
void CreateRequest<I>::mirror_image_enable() {
ldout(m_cct, 20) << this << " " << __func__ << dendl;

m_mirror_image_internal.state = cls::rbd::MIRROR_IMAGE_STATE_ENABLED;
if (m_non_primary_global_image_id.empty()) {
uuid_d uuid_gen;
uuid_gen.generate_random();
m_mirror_image_internal.global_image_id = uuid_gen.to_string();
} else {
m_mirror_image_internal.global_image_id = m_non_primary_global_image_id;
}

librados::ObjectWriteOperation op;
cls_client::mirror_image_set(&op, m_image_id, m_mirror_image_internal);

using klass = CreateRequest<I>;
librados::AioCompletion *comp =
create_rados_ack_callback<klass, &klass::handle_mirror_image_enable>(this);
int r = m_ioctx.aio_operate(RBD_MIRRORING, comp, &op);
assert(r == 0);
comp->release();
auto ctx = create_context_callback<
CreateRequest<I>, &CreateRequest<I>::handle_mirror_image_enable>(this);
auto req = mirror::EnableRequest<I>::create(m_ioctx, m_image_id,
m_non_primary_global_image_id,
m_op_work_queue, ctx);
req->send();
}

template<typename I>
Context *CreateRequest<I>::handle_mirror_image_enable(int *result) {
ldout(m_cct, 20) << __func__ << ": r=" << *result << dendl;

if (*result < 0) {
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result) << dendl;
lderr(m_cct) << "cannot enable mirroring: " << cpp_strerror(*result)
<< dendl;

m_r_saved = *result;
journal_remove();
return nullptr;
}

send_watcher_notification();
return nullptr;
}

// TODO: make this *really* async
template<typename I>
void CreateRequest<I>::send_watcher_notification() {
ldout(m_cct, 20) << this << " " << __func__ << dendl;

Context *ctx = new FunctionContext([this](int r) {
r = MirroringWatcher<>::notify_image_updated(
m_ioctx, cls::rbd::MIRROR_IMAGE_STATE_ENABLED,
m_image_id, m_mirror_image_internal.global_image_id);
handle_watcher_notify(r);
});

m_op_work_queue->queue(ctx, 0);
}

template<typename I>
void CreateRequest<I>::handle_watcher_notify(int r) {
ldout(m_cct, 20) << __func__ << ": r=" << r << dendl;

if (r < 0) {
// non fatal error -- watchers would cope up upon noticing missing
// updates. just log and move on...
ldout(m_cct, 10) << "failed to send update notification: " << cpp_strerror(r)
<< dendl;
} else {
ldout(m_cct, 20) << "image mirroring is enabled: global_id=" <<
m_mirror_image_internal.global_image_id << dendl;
}

complete(0);
return m_on_finish;
}

template<typename I>
Expand Down
21 changes: 7 additions & 14 deletions src/librbd/image/CreateRequest.h
View file
Open in desktop
Expand Up @@ -35,10 +35,12 @@ class CreateRequest {
const ImageOptions &image_options,
const std::string &non_primary_global_image_id,
const std::string &primary_mirror_uuid,
bool skip_mirror_enable,
ContextWQ *op_work_queue, Context *on_finish) {
return new CreateRequest(ioctx, image_name, image_id, size, image_options,
non_primary_global_image_id, primary_mirror_uuid,
op_work_queue, on_finish);
skip_mirror_enable, op_work_queue,
on_finish);
}

static int validate_order(CephContext *cct, uint8_t order);
Expand Down Expand Up @@ -79,14 +81,9 @@ class CreateRequest {
* | |\ JOURNAL CREATE .
* | | \ / | .
* v | *<------------/ v .
* | | FETCH MIRROR IMAGE v
* | | MIRROR IMAGE ENABLE .
* | | / | .
* | JOURNAL REMOVE<--------/ v .
* | \ MIRROR IMAGE ENABLE .
* | \ / | .
* | *<------------/ v .
* | NOTIFY WATCHERS .
* | | .
* | JOURNAL REMOVE*<-------/ | .
* | v .
* |_____________>___________________<finish> . . . . < . . . .
*
Expand All @@ -98,6 +95,7 @@ class CreateRequest {
const ImageOptions &image_options,
const std::string &non_primary_global_image_id,
const std::string &primary_mirror_uuid,
bool skip_mirror_enable,
ContextWQ *op_work_queue, Context *on_finish);

IoCtx m_ioctx;
Expand All @@ -115,6 +113,7 @@ class CreateRequest {
int64_t m_data_pool_id = -1;
const std::string m_non_primary_global_image_id;
const std::string m_primary_mirror_uuid;
bool m_skip_mirror_enable;
bool m_negotiate_features = false;

ContextWQ *m_op_work_queue;
Expand Down Expand Up @@ -157,15 +156,9 @@ class CreateRequest {
void journal_create();
Context *handle_journal_create(int *result);

void fetch_mirror_image();
Context *handle_fetch_mirror_image(int *result);

void mirror_image_enable();
Context *handle_mirror_image_enable(int *result);

void send_watcher_notification();
void handle_watcher_notify(int r);

void complete(int r);

// cleanup
Expand Down