Skip to content

Commit

Permalink
Add support to rados SD backend for libradosstriper.
Browse files Browse the repository at this point in the history
  • Loading branch information
Marco van Wieringen committed Feb 17, 2015
1 parent 07b877e commit 11fa5a7
Show file tree
Hide file tree
Showing 4 changed files with 221 additions and 8 deletions.
63 changes: 63 additions & 0 deletions autoconf/configure.in
View file
Expand Up @@ -3739,6 +3739,68 @@ fi
AC_SUBST(RADOS_INC)
AC_SUBST(RADOS_LIBS)

dnl
dnl Check for CEPH striping via libradosstriper.
dnl
RADOS_STRIPER_LIBS="-lradosstriper"
RADOS_STRIPER_INC=""
have_ceph_rados_striper=no
AC_ARG_WITH(rados-striper,
AC_HELP_STRING([--with-rados-striper@<:@=DIR@:>@], [Directory holding RADOS striper includes/libs]),
with_rados_striper_directory=$withval
)

if test "x${with_rados_striper_directory}" != "xyes" && test x"${with_rados_striper_directory}" != "x"; then
#
# Make sure the $with_rados_striper_directory also makes sense
#
if test -d "${with_rados_striper_directory}/lib" -a -d "${with_rados_striper_directory}/include"; then
RADOS_STRIPER_LIBS="-L${with_rados_striper_directory}/lib ${RADOS_STRIPER_LIBS}"
RADOS_STRIPER_INC="-I${with_rados_striper_directory}/include ${RADOS_STRIPER_INC}"
fi
fi

saved_LIBS="${LIBS}"
saved_CFLAGS="${CFLAGS}"
saved_CPPFLAGS="${CPPFLAGS}"
LIBS="${saved_LIBS} ${RADOS_STRIPER_LIBS}"
CFLAGS="${saved_CFLAGS} ${RADOS_STRIPER_INC}"
CPPFLAGS="${saved_CPPFLAGS} ${RADOS_STRIPER_INC}"

AC_CHECK_HEADER(radosstriper/libradosstriper.h)

AC_MSG_CHECKING(for rados_striper_create in radosstriper library)
AC_TRY_LINK(
[
#include <radosstriper/libradosstriper.h>
], [
rados_ioctx_t io_ctx;
rados_striper_t striper;

rados_striper_create(io_ctx, &striper);
], [
AC_MSG_RESULT(yes)
have_ceph_rados_striper="yes"
], [
AC_MSG_RESULT(no)
have_ceph_rados_striper="no"
]
)

LIBS="${saved_LIBS}"
CFLAGS="${saved_CFLAGS}"
CPPFLAGS="${saved_CPPFLAGS}"

if test "x${have_ceph_rados_striper}" = "xyes"; then
AC_DEFINE(HAVE_RADOS_STRIPER, 1, [Define to 1 if you have radosstriper lib])
else
RADOS_STRIPER_LIBS=""
RADOS_STRIPER_INC=""
fi

AC_SUBST(RADOS_STRIPER_INC)
AC_SUBST(RADOS_STRIPER_LIBS)

dnl
dnl Check for CEPH via libcephfs.
dnl
Expand Down Expand Up @@ -4511,6 +4573,7 @@ Configuration on `date`:
GLUSTERFS support: ${have_glusterfs}
DROPLET support: ${have_droplet}
CEPH RADOS support: ${have_ceph_rados}
RADOS striping support: ${have_ceph_rados_striper}
CEPHFS support: ${have_cephfs}
Python support: ${support_python} ${PYTHON_LIBS}
systemd support: ${support_systemd} ${SYSTEMD_UNITDIR}
Expand Down
4 changes: 2 additions & 2 deletions src/stored/backends/Makefile.in
View file
Expand Up @@ -20,12 +20,12 @@ dummy:
CEPHFS_INC = @CEPHFS_INC@
DROPLET_INC = @DROPLET_INC@
GLUSTER_INC = @GLUSTER_INC@
RADOS_INC = @RADOS_INC@
RADOS_INC = @RADOS_INC@ @RADOS_STRIPER_INC@

CEPHFS_LIBS = @CEPHFS_LIBS@
DROPLET_LIBS = @DROPLET_LIBS@
GLUSTER_LIBS = @GLUSTER_LIBS@
RADOS_LIBS = @RADOS_LIBS@
RADOS_LIBS = @RADOS_STRIPER_LIBS@ @RADOS_LIBS@

CHEPHFS_SRCS = cephfs_device.c
CHEPHFS_LOBJS = $(CHEPHFS_SRCS:.c=.lo)
Expand Down
150 changes: 144 additions & 6 deletions src/stored/backends/rados_device.c
View file
Expand Up @@ -30,6 +30,31 @@
#include "stored.h"
#include "rados_device.h"

/*
* Options that can be specified for this device type.
*/
enum device_option_type {
argument_none = 0,
argument_striped,
argument_stripe_unit,
argument_stripe_count
};

struct device_option {
const char *name;
enum device_option_type type;
int compare_size;
};

static device_option device_options[] = {
#ifdef HAVE_RADOS_STRIPER
{ "striped", argument_striped, 7 },
{ "stripe_unit=", argument_stripe_unit, 11 },
{ "stripe_count=", argument_stripe_count, 12 },
#endif
{ NULL, argument_none }
};

/*
* Open a volume using librados.
*/
Expand All @@ -41,7 +66,7 @@ int rados_device::d_open(const char *pathname, int flags, int mode)
berrno be;

if (!m_rados_configstring) {
char *bp;
char *bp, *next_option;

m_rados_configstring = bstrdup(dev_name);
bp = strchr(m_rados_configstring, ':');
Expand All @@ -54,6 +79,46 @@ int rados_device::d_open(const char *pathname, int flags, int mode)
*bp++ = '\0';
m_rados_conffile = m_rados_configstring;
m_rados_poolname = bp;

/*
* See if there are any options.
*/
bp = strchr(m_rados_poolname, ':');
if (bp) {
*bp++ = '\0';

while (bp) {
next_option = strchr(bp, ',');
if (next_option) {
*next_option++ = '\0';
}

for (int i = 0; device_options[i].name; i++) {
/*
* Try to find a matching device option.
*/
if (bstrncasecmp(bp, device_options[i].name, device_options[i].compare_size)) {
switch (device_options[i].type) {
#ifdef HAVE_RADOS_STRIPER
case argument_striped:
m_stripe_volume = true;
break;
case argument_stripe_unit:
m_stripe_unit = str_to_int64(bp + device_options[i].compare_size);
break;
case argument_stripe_count:
m_stripe_count = str_to_int64(bp + device_options[i].compare_size);
break;
#endif
default:
break;
}
}
}

bp = next_option;
}
}
}

if (!m_cluster_initialized) {
Expand Down Expand Up @@ -90,6 +155,31 @@ int rados_device::d_open(const char *pathname, int flags, int mode)
Emsg0(M_FATAL, 0, errmsg);
goto bail_out;
}

#ifdef HAVE_RADOS_STRIPER
if (m_stripe_volume) {
status = rados_striper_create(m_ctx, &m_striper);
if (status < 0) {
Mmsg2(errmsg, _("Unable to create RADOS striper object for pool %s: ERR=%s\n"), m_rados_poolname, be.bstrerror(-status));
Emsg0(M_FATAL, 0, errmsg);
goto bail_out;
}

status = rados_striper_set_object_layout_stripe_unit(m_striper, m_stripe_unit);
if (status < 0) {
Mmsg3(errmsg, _("Unable to set RADOS striper unit size to %d for pool %s: ERR=%s\n"), m_stripe_unit, m_rados_poolname, be.bstrerror(-status));
Emsg0(M_FATAL, 0, errmsg);
goto bail_out;
}

status = rados_striper_set_object_layout_stripe_count(m_striper, m_stripe_count);
if (status < 0) {
Mmsg3(errmsg, _("Unable to set RADOS striper stripe count to %d for pool %s: ERR=%s\n"), m_stripe_count, m_rados_poolname, be.bstrerror(-status));
Emsg0(M_FATAL, 0, errmsg);
goto bail_out;
}
}
#endif
}

/*
Expand All @@ -108,8 +198,17 @@ int rados_device::d_open(const char *pathname, int flags, int mode)
* Create an empty object.
* e.g. write one byte and then truncate it to zero bytes.
*/
rados_write(m_ctx, getVolCatName(), " ", 1, 0);
rados_trunc(m_ctx, getVolCatName(), 0);
#ifdef HAVE_RADOS_STRIPER
if (m_stripe_volume) {
rados_striper_write(m_ctx, getVolCatName(), " ", 1, 0);
rados_striper_trunc(m_ctx, getVolCatName(), 0);
} else {
#endif
rados_write(m_ctx, getVolCatName(), " ", 1, 0);
rados_trunc(m_ctx, getVolCatName(), 0);
#ifdef HAVE_RADOS_STRIPER
}
#endif
break;
default:
errno = -status;
Expand Down Expand Up @@ -144,7 +243,16 @@ ssize_t rados_device::d_read(int fd, void *buffer, size_t count)
if (m_ctx) {
int nr_read;

nr_read = rados_read(m_ctx, getVolCatName(), (char *)buffer, count, m_offset);
#ifdef HAVE_RADOS_STRIPER
if (m_striper) {
nr_read = rados_striper_read(m_striper, getVolCatName(), (char *)buffer, count, m_offset);
} else {
#endif
nr_read = rados_read(m_ctx, getVolCatName(), (char *)buffer, count, m_offset);
#ifdef HAVE_RADOS_STRIPER
}
#endif

if (nr_read >= 0) {
m_offset += nr_read;
return nr_read;
Expand All @@ -166,7 +274,16 @@ ssize_t rados_device::d_write(int fd, const void *buffer, size_t count)
if (m_ctx) {
int nr_written;

nr_written = rados_write(m_ctx, getVolCatName(), (char *)buffer, count, m_offset);
#ifdef HAVE_RADOS_STRIPER
if (m_striper) {
nr_written = rados_striper_write(m_striper, getVolCatName(), (char *)buffer, count, m_offset);
} else {
#endif
nr_written = rados_write(m_ctx, getVolCatName(), (char *)buffer, count, m_offset);
#ifdef HAVE_RADOS_STRIPER
}
#endif

if (nr_written >= 0) {
m_offset += nr_written;
return nr_written;
Expand All @@ -186,6 +303,12 @@ int rados_device::d_close(int fd)
* Destroy the IOcontext.
*/
if (m_ctx) {
#ifdef HAVE_RADOS_STRIPER
if (m_striper) {
rados_striper_destroy(m_striper);
m_striper = NULL;
}
#endif
rados_ioctx_destroy(m_ctx);
m_ctx = NULL;
} else {
Expand Down Expand Up @@ -236,7 +359,16 @@ bool rados_device::d_truncate(DCR *dcr)
time_t object_mtime;
berrno be;

status = rados_trunc(m_ctx, getVolCatName(), 0);
#ifdef HAVE_RADOS_STRIPER
if (m_stripe_volume) {
status = rados_striper_trunc(m_ctx, getVolCatName(), 0);
} else {
#endif
status = rados_trunc(m_ctx, getVolCatName(), 0);
#ifdef HAVE_RADOS_STRIPER
}
#endif

if (status < 0) {
Mmsg2(errmsg, _("Unable to truncate device %s. ERR=%s\n"), prt_name, be.bstrerror(-status));
Emsg0(M_FATAL, 0, errmsg);
Expand Down Expand Up @@ -287,6 +419,12 @@ rados_device::rados_device()
m_rados_poolname = NULL;
m_cluster_initialized = false;
m_ctx = NULL;
#ifdef HAVE_RADOS_STRIPER
m_stripe_volume = false;
m_stripe_unit = 0;
m_stripe_count = 0;
m_striper = NULL;
#endif
}

#ifdef HAVE_DYNAMIC_SD_BACKENDS
Expand Down
12 changes: 12 additions & 0 deletions src/stored/backends/rados_device.h
View file
Expand Up @@ -29,14 +29,26 @@

#include <rados/librados.h>

#ifdef HAVE_RADOS_STRIPER
#include <radosstriper/libradosstriper.h>
#endif

class rados_device: public DEVICE {
private:
char *m_rados_configstring;
char *m_rados_conffile;
char *m_rados_poolname;
bool m_cluster_initialized;
#ifdef HAVE_RADOS_STRIPER
bool m_stripe_volume;
uint32_t m_stripe_unit;
uint32_t m_stripe_count;
#endif
rados_t m_cluster;
rados_ioctx_t m_ctx;
#ifdef HAVE_RADOS_STRIPER
rados_striper_t m_striper;
#endif
boffset_t m_offset;

public:
Expand Down

0 comments on commit 11fa5a7

Please sign in to comment.