Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

rbd: parallelize "rbd ls -l" #15579

Merged
merged 1 commit into from Aug 8, 2017
Merged

rbd: parallelize "rbd ls -l" #15579

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
333 changes: 206 additions & 127 deletions src/tools/rbd/action/List.cc
View file
Open in desktop
Expand Up @@ -4,25 +4,166 @@
#include "tools/rbd/ArgumentTypes.h"
#include "tools/rbd/Shell.h"
#include "tools/rbd/Utils.h"
#include "include/Context.h"
#include "include/stringify.h"
#include "include/types.h"
#include "common/errno.h"
#include "common/Formatter.h"
#include "common/TextTable.h"
#include <iostream>
#include <boost/program_options.hpp>
#include "global/global_context.h"

namespace rbd {

namespace action {
namespace list {

namespace at = argument_types;
namespace po = boost::program_options;

int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag,
Formatter *f) {
enum WorkerState {
STATE_IDLE = 0,
STATE_OPENED,
STATE_DONE
} ;

struct WorkerEntry {
librbd::Image img;
librbd::RBD::AioCompletion* completion;
WorkerState state;
string name;

WorkerEntry() {
state = STATE_IDLE;
completion = nullptr;
}
};


int list_process_image(librados::Rados* rados, WorkerEntry* w, bool lflag, Formatter *f, TextTable &tbl)
{
int r = 0;
librbd::image_info_t info;
std::string pool, image, snap, parent;

// handle second-nth trips through loop
r = w->img.parent_info(&pool, &image, &snap);
if (r < 0 && r != -ENOENT)
return r;
bool has_parent = false;
if (r != -ENOENT) {
parent = pool + "/" + image + "@" + snap;
has_parent = true;
}

if (w->img.stat(info, sizeof(info)) < 0) {
return -EINVAL;
}

uint8_t old_format;
w->img.old_format(&old_format);

std::list<librbd::locker_t> lockers;
bool exclusive;
r = w->img.list_lockers(&lockers, &exclusive, NULL);
if (r < 0)
return r;
std::string lockstr;
if (!lockers.empty()) {
lockstr = (exclusive) ? "excl" : "shr";
}

if (f) {
f->open_object_section("image");
f->dump_string("image", w->name);
f->dump_unsigned("size", info.size);
if (has_parent) {
f->open_object_section("parent");
f->dump_string("pool", pool);
f->dump_string("image", image);
f->dump_string("snapshot", snap);
f->close_section();
}
f->dump_int("format", old_format ? 1 : 2);
if (!lockers.empty())
f->dump_string("lock_type", exclusive ? "exclusive" : "shared");
f->close_section();
} else {
tbl << w->name
<< stringify(si_t(info.size))
<< parent
<< ((old_format) ? '1' : '2')
<< "" // protect doesn't apply to images
<< lockstr
<< TextTable::endrow;
}

std::vector<librbd::snap_info_t> snaplist;
if (w->img.snap_list(snaplist) >= 0 && !snaplist.empty()) {
for (std::vector<librbd::snap_info_t>::iterator s = snaplist.begin();
s != snaplist.end(); ++s) {
bool is_protected;
bool has_parent = false;
parent.clear();
w->img.snap_set(s->name.c_str());
r = w->img.snap_is_protected(s->name.c_str(), &is_protected);
if (r < 0)
return r;
if (w->img.parent_info(&pool, &image, &snap) >= 0) {
parent = pool + "/" + image + "@" + snap;
has_parent = true;
}
if (f) {
f->open_object_section("snapshot");
f->dump_string("image", w->name);
f->dump_string("snapshot", s->name);
f->dump_unsigned("size", s->size);
if (has_parent) {
f->open_object_section("parent");
f->dump_string("pool", pool);
f->dump_string("image", image);
f->dump_string("snapshot", snap);
f->close_section();
}
f->dump_int("format", old_format ? 1 : 2);
f->dump_string("protected", is_protected ? "true" : "false");
f->close_section();
} else {
tbl << w->name + "@" + s->name
<< stringify(si_t(s->size))
<< parent
<< ((old_format) ? '1' : '2')
<< (is_protected ? "yes" : "")
<< "" // locks don't apply to snaps
<< TextTable::endrow;
}
}
}

return r < 0 ? r : 0;
}

int do_list(std::string &pool_name, bool lflag, int threads, Formatter *f) {
std::vector<WorkerEntry*> workers;
std::vector<std::string> names;
int r = rbd.list(io_ctx, names);
librados::Rados rados;
librbd::RBD rbd;
librados::IoCtx ioctx;

if (threads < 1) {
threads = 1;
}
if (threads > 32) {
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: if someone has a large enough cluster, why limit them to 32? I would just remove this guard.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried running it with 100 parallel jobs and it ate almost 140MB of memory, which is quite high, yet that much didn't help in any way. Anything above 150 caused it to crash in lockdep on vstart cluster and I expect it to cause a lot of other problems with absurd amounts of parallel jobs (think of "make -j 322" kind of typo), so IMHO it's better to cap it to some reasonable value now, than collect crash reports later.

threads = 32;
}

int r = utils::init(pool_name, &rados, &ioctx);
if (r < 0) {
return r;
}

r = rbd.list(ioctx, names);
if (r < 0)
return r;

Expand All @@ -32,9 +173,9 @@ int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag,
for (std::vector<std::string>::const_iterator i = names.begin();
i != names.end(); ++i) {
if (f)
f->dump_string("name", *i);
f->dump_string("name", *i);
else
std::cout << *i << std::endl;
std::cout << *i << std::endl;
}
if (f) {
f->close_section();
Expand All @@ -56,132 +197,78 @@ int do_list(librbd::RBD &rbd, librados::IoCtx& io_ctx, bool lflag,
tbl.define_column("LOCK", TextTable::LEFT, TextTable::LEFT);
}

std::string pool, image, snap, parent;

for (std::vector<std::string>::const_iterator i = names.begin();
i != names.end(); ++i) {
librbd::image_info_t info;
librbd::Image im;

r = rbd.open_read_only(io_ctx, im, i->c_str(), NULL);
// image might disappear between rbd.list() and rbd.open(); ignore
// that, warn about other possible errors (EPERM, say, for opening
// an old-format image, because you need execute permission for the
// class method)
if (r < 0) {
if (r != -ENOENT) {
std::cerr << "rbd: error opening " << *i << ": " << cpp_strerror(r)
<< std::endl;
}
// in any event, continue to next image
continue;
}

// handle second-nth trips through loop
parent.clear();
r = im.parent_info(&pool, &image, &snap);
if (r < 0 && r != -ENOENT)
goto out;
bool has_parent = false;
if (r != -ENOENT) {
parent = pool + "/" + image + "@" + snap;
has_parent = true;
}

if (im.stat(info, sizeof(info)) < 0) {
r = -EINVAL;
goto out;
}

uint8_t old_format;
im.old_format(&old_format);

std::list<librbd::locker_t> lockers;
bool exclusive;
r = im.list_lockers(&lockers, &exclusive, NULL);
if (r < 0)
goto out;
std::string lockstr;
if (!lockers.empty()) {
lockstr = (exclusive) ? "excl" : "shr";
}
for (int left = 0; left < std::min(threads, (int)names.size()); left++) {
workers.push_back(new WorkerEntry());
}

if (f) {
f->open_object_section("image");
f->dump_string("image", *i);
f->dump_unsigned("size", info.size);
if (has_parent) {
f->open_object_section("parent");
f->dump_string("pool", pool);
f->dump_string("image", image);
f->dump_string("snapshot", snap);
f->close_section();
auto i = names.begin();
while (true) {
size_t workers_idle = 0;
for (auto comp : workers) {
switch (comp->state) {
case STATE_DONE:
comp->completion->wait_for_complete();
comp->state = STATE_IDLE;
comp->completion->release();
comp->completion = nullptr;
// we want it to fall through in this case
case STATE_IDLE:
if (i == names.end()) {
workers_idle++;
continue;
}
comp->name = *i;
comp->completion = new librbd::RBD::AioCompletion(nullptr, nullptr);
r = rbd.aio_open_read_only(ioctx, comp->img, i->c_str(), NULL, comp->completion);
i++;
comp->state = STATE_OPENED;
break;
case STATE_OPENED:
comp->completion->wait_for_complete();
// image might disappear between rbd.list() and rbd.open(); ignore
// that, warn about other possible errors (EPERM, say, for opening
// an old-format image, because you need execute permission for the
// class method)
r = comp->completion->get_return_value();
comp->completion->release();
if (r < 0) {
if (r != -ENOENT) {
std::cerr << "rbd: error opening " << *i << ": " << cpp_strerror(r)
<< std::endl;
}
// in any event, continue to next image
comp->state = STATE_IDLE;
continue;
}
r = list_process_image(&rados, comp, lflag, f, tbl);
if (r < 0) {
std::cerr << "rbd: error processing image " << comp->name << ": " << cpp_strerror(r)
<< std::endl;
}
comp->completion = new librbd::RBD::AioCompletion(nullptr, nullptr);
r = comp->img.aio_close(comp->completion);
comp->state = STATE_DONE;
break;
}
f->dump_int("format", old_format ? 1 : 2);
if (!lockers.empty())
f->dump_string("lock_type", exclusive ? "exclusive" : "shared");
f->close_section();
} else {
tbl << *i
<< stringify(si_t(info.size))
<< parent
<< ((old_format) ? '1' : '2')
<< "" // protect doesn't apply to images
<< lockstr
<< TextTable::endrow;
}

std::vector<librbd::snap_info_t> snaplist;
if (im.snap_list(snaplist) >= 0 && !snaplist.empty()) {
for (std::vector<librbd::snap_info_t>::iterator s = snaplist.begin();
s != snaplist.end(); ++s) {
bool is_protected;
bool has_parent = false;
parent.clear();
im.snap_set(s->name.c_str());
r = im.snap_is_protected(s->name.c_str(), &is_protected);
if (r < 0)
goto out;
if (im.parent_info(&pool, &image, &snap) >= 0) {
parent = pool + "/" + image + "@" + snap;
has_parent = true;
}
if (f) {
f->open_object_section("snapshot");
f->dump_string("image", *i);
f->dump_string("snapshot", s->name);
f->dump_unsigned("size", s->size);
if (has_parent) {
f->open_object_section("parent");
f->dump_string("pool", pool);
f->dump_string("image", image);
f->dump_string("snapshot", snap);
f->close_section();
}
f->dump_int("format", old_format ? 1 : 2);
f->dump_string("protected", is_protected ? "true" : "false");
f->close_section();
} else {
tbl << *i + "@" + s->name
<< stringify(si_t(s->size))
<< parent
<< ((old_format) ? '1' : '2')
<< (is_protected ? "yes" : "")
<< "" // locks don't apply to snaps
<< TextTable::endrow;
}
}
if (workers_idle == workers.size()) {
break;
}
}

out:
if (f) {
f->close_section();
f->flush(std::cout);
} else if (!names.empty()) {
std::cout << tbl;
}

rados.shutdown();

for (auto comp : workers) {
delete comp;
}

return r < 0 ? r : 0;
}

Expand All @@ -203,15 +290,7 @@ int execute(const po::variables_map &vm) {
return r;
}

librados::Rados rados;
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You should be able to leave all of this logic for initializing the rados connection instead of creating your own solution

librados::IoCtx io_ctx;
r = utils::init(pool_name, &rados, &io_ctx);
if (r < 0) {
return r;
}

librbd::RBD rbd;
r = do_list(rbd, io_ctx, vm["long"].as<bool>(), formatter.get());
r = do_list(pool_name, vm["long"].as<bool>(), g_conf->rbd_concurrent_management_ops, formatter.get());
if (r < 0) {
std::cerr << "rbd: list: " << cpp_strerror(r) << std::endl;
return r;
Expand Down