Skip to content

Commit

Permalink
Merge pull request #7127 from trociny/wip-rbd-nbd-misc
Browse files Browse the repository at this point in the history 
rbd-nbd: small improvements in logging and forking

Reviewed-by: Josh Durgin <jdurgin@redhat.com>
  • Loading branch information
@jdurgin
jdurgin committed Jan 6, 2016
2 parents 31131cf + a286e7c commit fec0ce4
Showing 1 changed file with 145 additions and 53 deletions.
198 changes: 145 additions & 53 deletions src/tools/rbd_nbd/rbd-nbd.cc
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
// vim: ts=8 sw=2 smarttab

#include "include/int_types.h"

#include <stdio.h>
Expand All @@ -20,6 +23,7 @@

#include "mon/MonClient.h"
#include "common/config.h"
#include "common/dout.h"

#include "common/errno.h"
#include "common/module.h"
Expand All @@ -31,18 +35,23 @@
#include "include/rados/librados.hpp"
#include "include/rbd/librbd.hpp"

#define dout_subsys ceph_subsys_rbd
#undef dout_prefix
#define dout_prefix *_dout << "rbd-nbd: "

static void usage()
{
std::cout << "Usage: rbd-nbd [options] map <image-or-snap-spec> Map a image to nbd device\n"
<< " unmap <device path> Unmap nbd device\n"
<< " list-mapped List mapped nbd devices\n"
<< "Options: --device <device path> Specify nbd device path\n"
<< " --read-only Map readonly\n"
<< " --nbds_max <limit> Override for module param\n"
<< "Options:\n"
<< " --device <device path> Specify nbd device path\n"
<< " --read-only Map readonly\n"
<< " --nbds_max <limit> Override for module param\n"
<< std::endl;
generic_server_usage();
}

static Preforker forker;
static std::string devpath, poolname("rbd"), imgname, snapname;
static bool readonly = false;
static int nbds_max = 0;
Expand Down Expand Up @@ -100,6 +109,8 @@ class NBDServer
{}
};

friend std::ostream &operator<<(std::ostream &os, const IOContext &ctx);

Mutex lock;
Cond cond;
xlist<IOContext*> io_pending;
Expand Down Expand Up @@ -155,9 +166,18 @@ class NBDServer

IOContext *ctx = reinterpret_cast<IOContext *>(arg);
int ret = aio_completion->get_return_value();
if (ret > 0)
ret = 0;
ctx->reply.error = htonl(ret);

dout(20) << __func__ << ": " << *ctx << dendl;

if (ret < 0) {
ctx->reply.error = htonl(-ret);
} else if (ret != static_cast<int>(ctx->request.len)) {
derr << __func__ << ": " << *ctx << ": unexpected return value: " << ret
<< " (" << ctx->request.len << " expected)" << dendl;
ctx->reply.error = htonl(EIO);
} else {
ctx->reply.error = htonl(0);
}
ctx->server->io_finish(ctx);

aio_completion->release();
Expand All @@ -168,11 +188,20 @@ class NBDServer
while (!terminated.read()) {
ceph::unique_ptr<IOContext> ctx(new IOContext());
ctx->server = this;
if (safe_read_exact(fd, &ctx->request, sizeof(struct nbd_request)) < 0)
return;

if (ctx->request.magic != htonl(NBD_REQUEST_MAGIC))
return;
dout(20) << __func__ << ": waiting for nbd request" << dendl;

int r = safe_read_exact(fd, &ctx->request, sizeof(struct nbd_request));
if (r < 0) {
derr << "failed to read nbd request header: " << cpp_strerror(errno)
<< dendl;
return;
}

if (ctx->request.magic != htonl(NBD_REQUEST_MAGIC)) {
derr << "invalid nbd request header" << dendl;
return;
}

ctx->request.from = ntohll(ctx->request.from);
ctx->request.type = ntohl(ctx->request.type);
Expand All @@ -183,14 +212,21 @@ class NBDServer

ctx->command = ctx->request.type & 0x0000ffff;

dout(20) << *ctx << ": start" << dendl;

switch (ctx->command)
{
case NBD_CMD_DISC:
dout(0) << "disconnect request received" << dendl;
return;
case NBD_CMD_WRITE:
bufferptr ptr(ctx->request.len);
if (safe_read_exact(fd, ptr.c_str(), ctx->request.len) < 0)
r = safe_read_exact(fd, ptr.c_str(), ctx->request.len);
if (r < 0) {
derr << *ctx << ": failed to read nbd request data: "
<< cpp_strerror(errno) << dendl;
return;
}
ctx->data.push_back(ptr);
break;
}
Expand All @@ -213,25 +249,42 @@ class NBDServer
image.aio_discard(pctx->request.from, pctx->request.len, c);
break;
default:
derr << *pctx << ": invalid request command" << dendl;
return;
}
}
dout(20) << __func__ << ": terminated" << dendl;
}

void writer_entry()
{
while (!terminated.read()) {
dout(20) << __func__ << ": waiting for io request" << dendl;
ceph::unique_ptr<IOContext> ctx(wait_io_finish());
if (!ctx)
if (!ctx) {
dout(20) << __func__ << ": no io requests, terminating" << dendl;
return;
}

dout(20) << __func__ << ": got: " << *ctx << dendl;

if (safe_write(fd, &ctx->reply, sizeof(struct nbd_reply)) < 0)
int r = safe_write(fd, &ctx->reply, sizeof(struct nbd_reply));
if (r < 0) {
derr << *ctx << ": failed to write reply header: " << cpp_strerror(r)
<< dendl;
return;
}
if (ctx->command == NBD_CMD_READ && ctx->reply.error == htonl(0)) {
if (ctx->data.write_fd(fd) < 0)
r = ctx->data.write_fd(fd);
if (r < 0) {
derr << *ctx << ": failed to write replay data: " << cpp_strerror(r)
<< dendl;
return;
}
}
dout(20) << *ctx << ": finish" << dendl;
}
dout(20) << __func__ << ": terminated" << dendl;
}

class ThreadHelper : public Thread
Expand Down Expand Up @@ -260,6 +313,8 @@ class NBDServer
void start()
{
if (!started) {
dout(10) << __func__ << ": starting" << dendl;

started = true;

reader_thread.create();
Expand All @@ -270,6 +325,8 @@ class NBDServer
void stop()
{
if (started) {
dout(10) << __func__ << ": terminating" << dendl;

shutdown();

reader_thread.join();
Expand All @@ -287,6 +344,34 @@ class NBDServer
}
};

std::ostream &operator<<(std::ostream &os, const NBDServer::IOContext &ctx) {

os << "[" << std::hex << ntohll(*((uint64_t *)ctx.request.handle));

switch (ctx.command)
{
case NBD_CMD_WRITE:
os << " WRITE ";
break;
case NBD_CMD_READ:
os << " READ ";
break;
case NBD_CMD_FLUSH:
os << " FLUSH ";
break;
case NBD_CMD_TRIM:
os << " TRIM ";
break;
default:
os << " UNKNOW(" << ctx.command << ") ";
break;
}

os << ctx.request.from << "~" << ctx.request.len << " "
<< ntohl(ctx.reply.error) << "]";

return os;
}

class NBDWatchCtx : public librados::WatchCtx2
{
Expand Down Expand Up @@ -322,11 +407,11 @@ class NBDWatchCtx : public librados::WatchCtx2

if (new_size != size) {
if (ioctl(fd, BLKFLSBUF, NULL) < 0)
std::cerr << "rbd-nbd: invalidate page cache failed status: " << cpp_strerror(errno) << std::endl;
derr << "invalidate page cache failed: " << cpp_strerror(errno) << dendl;
if (ioctl(fd, NBD_SET_SIZE, new_size) < 0)
std::cerr << "rbd-nbd: resize failed status: " << cpp_strerror(errno) << std::endl;
derr << "resize failed: " << cpp_strerror(errno) << dendl;
if (image.invalidate_cache() < 0)
std::cerr << "rbd-nbd: invalidate rbd cache failed" << std::endl;
derr << "invalidate rbd cache failed" << dendl;
size = new_size;
}
}
Expand Down Expand Up @@ -377,11 +462,31 @@ static int do_map()

int fd[2];
int nbd;
int null_fd = -1;

uint8_t old_format;
librbd::image_info_t info;

Preforker forker;

if (global_init_prefork(g_ceph_context) >= 0) {
std::string err;
if (forker.prefork(err) < 0) {
cerr << err << std::endl;
return EXIT_FAILURE;
}

if (forker.is_parent()) {
if (forker.parent_wait(err) < 0) {
cerr << err << std::endl;
return EXIT_FAILURE;
}
return 0;
}
}

common_init_finish(g_ceph_context);
global_init_chdir(g_ceph_context);

if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) == -1) {
r = -errno;
goto close_ret;
Expand Down Expand Up @@ -503,22 +608,12 @@ static int do_map()
if (r < 0)
goto close_nbd;

if (g_conf->daemonize) {
r = open("/dev/null", O_RDWR);
if (r < 0)
goto close_watcher;
null_fd = r;
}

cout << devpath << std::endl;

if (g_conf->daemonize) {
forker.daemonize();

::dup2(null_fd, STDIN_FILENO);
::dup2(null_fd, STDOUT_FILENO);
::dup2(null_fd, STDERR_FILENO);
close(null_fd);
global_init_postfork_start(g_ceph_context);
global_init_postfork_finish(g_ceph_context);
}

{
Expand All @@ -529,7 +624,6 @@ static int do_map()
server.stop();
}

close_watcher:
io_ctx.unwatch2(watcher);
}

Expand All @@ -546,11 +640,16 @@ static int do_map()
image.close();
io_ctx.close();
rados.shutdown();

forker.exit(r < 0 ? EXIT_FAILURE : 0);
// Unreachable;
return r;
}

static int do_unmap()
{
common_init_finish(g_ceph_context);

int nbd = open_device(devpath.c_str());
if (nbd < 0) {
cerr << "rbd-nbd: failed to open device: " << devpath << std::endl;
Expand Down Expand Up @@ -585,14 +684,19 @@ static int parse_imgpath(const std::string &imgpath)
return 0;
}

static void list_mapped_devices()
static int do_list_mapped_devices()
{
char path[64];
int m = 0;
int fd[2];

if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) == -1)
return;
common_init_finish(g_ceph_context);

if (socketpair(AF_UNIX, SOCK_STREAM, 0, fd) == -1) {
int r = -errno;
cerr << "rbd-nbd: socketpair failed: " << cpp_strerror(-r) << std::endl;
return r;
}

while (true) {
snprintf(path, sizeof(path), "/dev/nbd%d", m);
Expand All @@ -609,6 +713,8 @@ static void list_mapped_devices()

close(fd[0]);
close(fd[1]);

return 0;
}

static int rbd_nbd(int argc, const char *argv[])
Expand Down Expand Up @@ -692,8 +798,6 @@ static int rbd_nbd(int argc, const char *argv[])

switch (cmd) {
case Connect:
common_init_finish(g_ceph_context);

if (imgname.empty()) {
cerr << "rbd-nbd: image name was not specified" << std::endl;
return EXIT_FAILURE;
Expand All @@ -709,7 +813,9 @@ static int rbd_nbd(int argc, const char *argv[])
return EXIT_FAILURE;
break;
case List:
list_mapped_devices();
r = do_list_mapped_devices();
if (r < 0)
return EXIT_FAILURE;
break;
default:
usage();
Expand All @@ -721,19 +827,5 @@ static int rbd_nbd(int argc, const char *argv[])

int main(int argc, const char *argv[])
{
std::string err;

if (forker.prefork(err) < 0) {
cerr << err << std::endl;
return EXIT_FAILURE;
}

if (forker.is_child()) {
forker.exit(rbd_nbd(argc, argv));
} else if (forker.parent_wait(err) < 0) {
cerr << err << std::endl;
return EXIT_FAILURE;
} else {
return 0;
}
return rbd_nbd(argc, argv);
}

0 comments on commit fec0ce4

Please sign in to comment.