Merge pull request #6657 from kylinstorage/wip-rbd-nbd

rbd-nbd: network block device (NBD) support for RBD 

Reviewed-by: Jason Dillaman <dillaman@redhat.com>

ceph.spec.in

@@ -253,6 +253,15 @@ Requires:	librbd1 = %{epoch}:%{version}-%{release}
 %description -n rbd-fuse
 FUSE based client to map Ceph rbd images to files
 
+%package -n rbd-nbd
+Summary:	Ceph RBD client base on NBD
+Group:		System Environment/Base
+Requires:	%{name}
+Requires:	librados2 = %{epoch}:%{version}-%{release}
+Requires:	librbd1 = %{epoch}:%{version}-%{release}
+%description -n rbd-nbd
+NBD based client to map Ceph rbd images to local device
+
 %package radosgw
 Summary:	Rados REST gateway
 Group:		Development/Libraries
@@ -969,6 +978,12 @@ fi
 %{_bindir}/rbd-fuse
 %{_mandir}/man8/rbd-fuse.8*
 
+#################################################################################
+%files -n rbd-nbd
+%defattr(-,root,root,-)
+%{_bindir}/rbd-nbd
+%{_mandir}/man8/rbd-nbd.8*
+
 #################################################################################
 %files radosgw
 %defattr(-,root,root,-)

debian/control

@@ -190,6 +190,33 @@ Description: debugging symbols for rbd-fuse
  .
  This package contains the debugging symbols for rbd-fuse.
 
+Package: rbd-nbd
+Architecture: linux-any
+Depends: ${misc:Depends}, ${shlibs:Depends}
+Description: NBD-based rbd client for the Ceph distributed file system
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.  This is a
+ NBD-based client that allows one to map Ceph rbd images as local
+ block device.
+ .
+ NBD base client that allows one to map Ceph rbd images as local
+ block device.
+
+Package: rbd-nbd-dbg
+Architecture: linux-any
+Section: debug
+Priority: extra
+Depends: rbd-nbd (= ${binary:Version}), ${misc:Depends}
+Description: debugging symbols for rbd-nbd
+ Ceph is a massively scalable, open-source, distributed
+ storage system that runs on commodity hardware and delivers object,
+ block and file system storage.  This is a
+ NBD-based client that allows one to map Ceph rbd images as local
+ block device.
+ .
+ This package contains the debugging symbols for rbd-nbd.
+
 Package: ceph-common
 Architecture: linux-any
 Depends: librbd1 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends},

debian/rbd-nbd.install

@@ -0,0 +1,2 @@
+usr/bin/rbd-nbd
+usr/share/man/man8/rbd-nbd.8

debian/rules

@@ -161,6 +161,7 @@ binary-arch: build install
 	dh_strip -pceph-mds --dbg-package=ceph-mds-dbg
 	dh_strip -pceph-fuse --dbg-package=ceph-fuse-dbg
 	dh_strip -prbd-fuse --dbg-package=rbd-fuse-dbg
+	dh_strip -prbd-nbd --dbg-package=rbd-nbd-dbg
 	dh_strip -pceph-common --dbg-package=ceph-common-dbg
 	dh_strip -pceph-fs-common --dbg-package=ceph-fs-common-dbg
 	dh_strip -plibrados2 --dbg-package=librados2-dbg

doc/Makefile.am

@@ -27,6 +27,7 @@ EXTRA_DIST = \
 	man/8/radosgw.rst	\
 	man/8/rados.rst	\
 	man/8/rbd-fuse.rst	\
+	man/8/rbd-nbd.rst	\
 	man/8/rbd-replay-many.rst	\
 	man/8/rbd-replay-prep.rst	\
 	man/8/rbd-replay.rst	\

doc/man/8/rbd-nbd.rst

@@ -0,0 +1,55 @@
+:orphan:
+
+=========================================
+ rbd-nbd -- map rbd images to nbd device
+=========================================
+
+.. program:: rbd-nbd
+
+Synopsis
+========
+
+| **rbd-nbd** [-c conf] [--nbds_max *limit*] [--read-only] [--device *nbd device*] map *image-spec* | *snap-spec*
+| **rbd-nbd** unmap *nbd device*
+| **rbd-nbd** list-mapped
+
+Description
+===========
+
+**rbd-nbd** is a client for RADOS block device (rbd) images like rbd kernel module.
+It will map a rbd image to a nbd (Network Block Device) device, allowing access it
+as regular local block device.
+
+Options
+=======
+
+.. option:: -c ceph.conf
+
+   Use *ceph.conf* configuration file instead of the default
+   ``/etc/ceph/ceph.conf`` to determine monitor addresses during startup.
+
+.. option:: --nbds_max *limit*
+
+   Override the parameter of NBD kernel module when modprobe, used to
+   limit the count of nbd device.
+
+Image and snap specs
+====================
+
+| *image-spec* is [*pool-name*]/*image-name*
+| *snap-spec*  is [*pool-name*]/*image-name*\ @\ *snap-name*
+
+The default for *pool-name* is "rbd".  If an image name contains a slash
+character ('/'), *pool-name* is required.
+
+Availability
+============
+
+**rbd-nbd** is part of Ceph, a massively scalable, open-source, distributed storage system. Please refer to
+the Ceph documentation at http://ceph.com/docs for more information.
+
+
+See also
+========
+
+:doc:`rbd <rbd>`\(8)

doc/man/8/rbd.rst

@@ -321,6 +321,15 @@ Commands
 :command:`showmapped`
   Show the rbd images that are mapped via the rbd kernel module.
 
+:command:`nbd map` [--device *device-path*] [--read-only] *image-spec* | *snap-spec*
+  Maps the specified image to a block device via the rbd-nbd tool.
+
+:command:`nbd unmap` *device-path*
+  Unmaps the block device that was mapped via the rbd-nbd tool.
+
+:command:`nbd list`
+  Show the list of used nbd devices via the rbd-nbd tool.
+
 :command:`status` *image-spec*
   Show the status of the image, including which clients have it open.
 

doc/rbd/rbd.rst

@@ -51,6 +51,7 @@ devices simultaneously.
 	CloudStack <rbd-cloudstack>
 	Manpage rbd <../../man/8/rbd>
 	Manpage rbd-fuse <../../man/8/rbd-fuse>
+	Manpage rbd-nbd <../../man/8/rbd-nbd>
 	Manpage ceph-rbdnamer <../../man/8/ceph-rbdnamer>
 	RBD Replay <rbd-replay>
 	Manpage rbd-replay-prep <../../man/8/rbd-replay-prep>

man/CMakeLists.txt

@@ -26,6 +26,7 @@ install(FILES
 	ceph-rbdnamer.8
 	ceph-post-file.8
 	rbd-fuse.8
+	rbd-nbd.8
 	rbd-replay.8
 	rbd-replay-prep.8
   DESTINATION ${CEPH_MAN_DIR}/man8)

man/Makefile-client.am

@@ -16,6 +16,7 @@ endif
 if WITH_RBD
 dist_man_MANS += \
 	ceph-rbdnamer.8 \
+	rbd-nbd.8 \
 	rbd-replay.8 \
 	rbd-replay-many.8 \
 	rbd-replay-prep.8

qa/workunits/rbd/rbd-nbd.sh

@@ -0,0 +1,42 @@
+#!/bin/bash -ex
+
+pool=rbd
+gen=$pool/gen
+data=testfile
+size=64
+dev=/dev/nbd0
+
+mkdir -p rbd_nbd_test
+pushd rbd_nbd_test
+
+function expect_false()
+{
+  if "$@"; then return 1; else return 0; fi
+}
+
+rbd remove $gen || true
+rbd-nbd unmap $dev || true
+
+#read test
+dd if=/dev/urandom of=$data bs=1M count=$size
+rbd --no-progress import $data $gen
+rbd-nbd --device $dev map $gen
+[ "`dd if=$data bs=1M | md5sum`" != "`dd if=$dev bs=1M | md5sum`" ] && false
+
+#write test
+dd if=/dev/urandom of=$data bs=1M count=$size
+dd if=$data of=$dev bs=1M
+sync
+[ "`dd if=$data bs=1M | md5sum`" != "`rbd --no-progress export $gen - | md5sum`" ] && false
+
+#trim test
+mkfs.ext4 $dev # better idea?
+sync
+info=`rbd du $gen | tail -n 1`
+[ "`echo $info | awk '{print $2}'`" == "`echo $info | awk '{print $3}'`" ] && false
+
+rbd-nbd unmap $dev
+popd
+rm -rf rbd_nbd_test
+
+echo OK

src/.gitignore

@@ -74,6 +74,7 @@ Makefile
 /radosgw-admin
 /radosgw-object-expirer
 /rbd
+/rbd-nbd
 /rbd-fuse
 /rbd-replay
 /rbd-replay-prep

src/common/SubProcess.h

@@ -30,6 +30,7 @@
 
 #include <sstream>
 #include <vector>
+#include <iostream>
 
 #include <include/assert.h>
 #include <common/errno.h>
@@ -40,7 +41,7 @@
  *
  * Example:
  *
- *   SubProcess cat("cat", true, true);
+ *   SubProcess cat("cat", SubProcess::PIPE, SubProcess::PIPE);
  *   if (cat.spawn() != 0) {
  *     std::cerr << "cat failed: " << cat.err() << std::endl;
  *     return false;
@@ -56,8 +57,16 @@
 
 class SubProcess {
 public:
-  SubProcess(const char *cmd, bool pipe_stdin = false, bool pipe_stdout = false,
-	     bool pipe_stderr = false);
+  enum std_fd_op{
+    KEEP,
+    CLOSE,
+    PIPE
+  };
+public:
+  SubProcess(const char *cmd,
+             std_fd_op stdin_op = CLOSE,
+             std_fd_op stdout_op = CLOSE,
+             std_fd_op stderr_op = CLOSE);
   virtual ~SubProcess();
 
   void add_cmd_args(const char *arg, ...);
@@ -90,9 +99,9 @@ class SubProcess {
 protected:
   std::string cmd;
   std::vector<std::string> cmd_args;
-  bool pipe_stdin;
-  bool pipe_stdout;
-  bool pipe_stderr;
+  std_fd_op stdin_op;
+  std_fd_op stdout_op;
+  std_fd_op stderr_op;
   int stdin_pipe_out_fd;
   int stdout_pipe_in_fd;
   int stderr_pipe_in_fd;
@@ -102,8 +111,8 @@ class SubProcess {
 
 class SubProcessTimed : public SubProcess {
 public:
-  SubProcessTimed(const char *cmd, bool pipe_stdin = false,
-		  bool pipe_stdout = false, bool pipe_stderr = false,
+  SubProcessTimed(const char *cmd, std_fd_op stdin_op = CLOSE,
+		  std_fd_op stdout_op = CLOSE, std_fd_op stderr_op = CLOSE,
 		  int timeout = 0, int sigkill = SIGKILL);
 
 protected:
@@ -114,12 +123,12 @@ class SubProcessTimed : public SubProcess {
   int sigkill;
 };
 
-SubProcess::SubProcess(const char *cmd_, bool use_stdin, bool use_stdout, bool use_stderr) :
+SubProcess::SubProcess(const char *cmd_, std_fd_op stdin_op_, std_fd_op stdout_op_, std_fd_op stderr_op_) :
   cmd(cmd_),
   cmd_args(),
-  pipe_stdin(use_stdin),
-  pipe_stdout(use_stdout),
-  pipe_stderr(use_stderr),
+  stdin_op(stdin_op_),
+  stdout_op(stdout_op_),
+  stderr_op(stderr_op_),
   stdin_pipe_out_fd(-1),
   stdout_pipe_in_fd(-1),
   stderr_pipe_in_fd(-1),
@@ -155,21 +164,21 @@ void SubProcess::add_cmd_arg(const char *arg) {
 
 int SubProcess::get_stdin() const {
   assert(is_spawned());
-  assert(pipe_stdin);
+  assert(stdin_op == PIPE);
 
   return stdin_pipe_out_fd;
 }
 
 int SubProcess::get_stdout() const {
   assert(is_spawned());
-  assert(pipe_stdout);
+  assert(stdout_op == PIPE);
 
   return stdout_pipe_in_fd;
 }
 
 int SubProcess::get_stderr() const {
   assert(is_spawned());
-  assert(pipe_stderr);
+  assert(stderr_op == PIPE);
 
   return stderr_pipe_in_fd;
 }
@@ -184,21 +193,21 @@ void SubProcess::close(int &fd) {
 
 void SubProcess::close_stdin() {
   assert(is_spawned());
-  assert(pipe_stdin);
+  assert(stdin_op == PIPE);
 
   close(stdin_pipe_out_fd);
 }
 
 void SubProcess::close_stdout() {
   assert(is_spawned());
-  assert(pipe_stdout);
+  assert(stdout_op == PIPE);
 
   close(stdout_pipe_in_fd);
 }
 
 void SubProcess::close_stderr() {
   assert(is_spawned());
-  assert(pipe_stderr);
+  assert(stderr_op == PIPE);
 
   close(stderr_pipe_in_fd);
 }
@@ -247,9 +256,9 @@ int SubProcess::spawn() {
 
   int ret = 0;
 
-  if ((pipe_stdin  && ::pipe(ipipe) == -1) ||
-      (pipe_stdout && ::pipe(opipe) == -1) ||
-      (pipe_stderr && ::pipe(epipe) == -1)) {
+  if ((stdin_op == PIPE  && ::pipe(ipipe) == -1) ||
+      (stdout_op == PIPE && ::pipe(opipe) == -1) ||
+      (stderr_op == PIPE && ::pipe(epipe) == -1)) {
     ret = -errno;
     errstr << "pipe failed: " << cpp_strerror(errno);
     goto fail;
@@ -290,11 +299,11 @@ int SubProcess::spawn() {
     if (maxfd == -1)
       maxfd = 16384;
     for (int fd = 0; fd <= maxfd; fd++) {
-      if (fd == STDIN_FILENO && pipe_stdin)
+      if (fd == STDIN_FILENO && stdin_op != CLOSE)
 	continue;
-      if (fd == STDOUT_FILENO && pipe_stdout)
+      if (fd == STDOUT_FILENO && stdout_op != CLOSE)
 	continue;
-      if (fd == STDERR_FILENO && pipe_stderr)
+      if (fd == STDERR_FILENO && stderr_op != CLOSE)
 	continue;
       ::close(fd);
     }
@@ -363,10 +372,10 @@ int SubProcess::join() {
   return EXIT_FAILURE;
 }
 
-SubProcessTimed::SubProcessTimed(const char *cmd, bool pipe_stdin,
-				 bool pipe_stdout, bool pipe_stderr,
+SubProcessTimed::SubProcessTimed(const char *cmd, std_fd_op stdin_op,
+				 std_fd_op stdout_op, std_fd_op stderr_op,
 				 int timeout_, int sigkill_) :
-  SubProcess(cmd, pipe_stdin, pipe_stdout, pipe_stderr),
+  SubProcess(cmd, stdin_op, stdout_op, stderr_op),
   timeout(timeout_),
   sigkill(sigkill_) {
 }