rbd_dev: update device size if rbd resized

include/ubbd_dev.h

@@ -44,7 +44,7 @@ struct ubbd_dev_features {
 struct ubbd_device;
 struct ubbd_dev_ops {
 	struct ubbd_device* (*create) (struct __ubbd_dev_info *info);
-	int (*init) (struct ubbd_device *ubbd_dev);
+	int (*init) (struct ubbd_device *ubbd_dev, bool reopen);
 	void (*release) (struct ubbd_device *ubbd_dev);
 	int (*before_dev_remove) (struct ubbd_device *ubbd_dev);
 	int (*post_disk_added) (struct ubbd_device *ubbd_dev);
@@ -127,7 +127,7 @@ void ubbd_dev_put(struct ubbd_device *ubbd_dev);
 
 struct ubbd_device *find_ubbd_dev(int dev_id);
 struct ubbd_device *ubbd_dev_create(struct ubbd_dev_info *info, bool force);
-int ubbd_dev_init(struct ubbd_device *ubbd_dev);
+int ubbd_dev_init(struct ubbd_device *ubbd_dev, bool reopen);
 struct ubbd_device *ubbd_cache_dev_create(struct ubbd_dev_info *backing_dev_info,
 		struct ubbd_dev_info *cache_dev_info, int cache_mode, bool force);
 int ubbd_dev_restart(struct ubbd_device *ubbd_dev, int restart_mode);

include/ubbd_netlink.h

@@ -26,6 +26,7 @@ struct ubbd_nl_req {
 		} remove_opts;
 		struct config_options {
 			int data_pages_reserve_percnt;
+			uint64_t dev_size;
 		} config_opts;
 	} req_opts;
 	struct ubbd_device *ubbd_dev;
@@ -49,7 +50,8 @@ int ubbd_nl_req_add_dev(struct ubbd_device *ubbd_dev, struct context *ctx);
 int ubbd_nl_req_add_disk(struct ubbd_device *ubbd_dev, struct context *ctx);
 int ubbd_nl_req_remove_disk(struct ubbd_device *ubbd_dev, bool force, struct context *ctx);
 int ubbd_nl_req_remove_dev(struct ubbd_device *ubbd_dev, struct context *ctx);
-int ubbd_nl_req_config(struct ubbd_device *ubbd_dev, int data_pages_reserve_percnt, struct context *ctx);
+int ubbd_nl_req_config(struct ubbd_device *ubbd_dev, int data_pages_reserve_percnt,
+		uint64_t dev_size, struct context *ctx);
 int ubbd_nl_start_thread(void);
 void ubbd_nl_stop_thread(void);
 int ubbd_nl_wait_thread(void);

include/ubbd_rbd.h

@@ -18,6 +18,8 @@ struct ubbd_rbd_conn {
 	rbd_image_t image;
 	uint64_t flags;
 	int io_timeout;
+
+	uint64_t update_handle;
 };
 
 int ubbd_rbd_conn_open(struct ubbd_rbd_conn *rbd_conn);

lib/ubbd_devs/ubbd_cache_dev.c

@@ -3,16 +3,16 @@
 
 // cache ops
 
-static int cache_dev_init(struct ubbd_device *ubbd_dev)
+static int cache_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	struct ubbd_cache_device *cache_dev = CACHE_DEV(ubbd_dev);
 	int ret;
 
-	ret = ubbd_dev_init(cache_dev->backing_device);
+	ret = ubbd_dev_init(cache_dev->backing_device, reopen);
 	if (ret)
 		return ret;
 
-	ret = ubbd_dev_init(cache_dev->cache_device);
+	ret = ubbd_dev_init(cache_dev->cache_device, reopen);
 	if (ret)
 		return ret;
 

lib/ubbd_devs/ubbd_dev.c

@@ -170,11 +170,11 @@ struct ubbd_device *ubbd_dev_create(struct ubbd_dev_info *info, bool force)
 }
 
 
-int ubbd_dev_init(struct ubbd_device *ubbd_dev)
+int ubbd_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	int ret = 0;
 
-	ret = ubbd_dev->dev_ops->init(ubbd_dev);
+	ret = ubbd_dev->dev_ops->init(ubbd_dev, reopen);
 	if (ret)
 		goto out;
 
@@ -670,7 +670,7 @@ int ubbd_dev_add(struct ubbd_device *ubbd_dev, struct context *ctx)
 	int ret;
 
 	pthread_mutex_lock(&ubbd_dev->lock);
-	ret = ubbd_dev_init(ubbd_dev);
+	ret = ubbd_dev_init(ubbd_dev, false);
 	if (ret) {
 		goto release_dev;
 	}
@@ -842,7 +842,7 @@ int ubbd_dev_config(struct ubbd_device *ubbd_dev, int data_pages_reserve_percnt,
 	if (!config_ctx)
 		return -ENOMEM;
 
-	ret = ubbd_nl_req_config(ubbd_dev, data_pages_reserve_percnt, config_ctx);
+	ret = ubbd_nl_req_config(ubbd_dev, data_pages_reserve_percnt, 0, config_ctx);
 	pthread_mutex_unlock(&ubbd_dev->lock);
 	if (ret)
 		context_free(config_ctx);
@@ -994,6 +994,12 @@ static int reopen_dev(struct ubbd_nl_dev_status *dev_status,
 	ubbd_dev->new_backend_id = dev_conf->new_backend_id;
 	free(dev_conf);
 
+	ret = ubbd_dev_init(ubbd_dev, true);
+	if (ret) {
+		ubbd_dev->status = UBBD_DEV_USTATUS_ERROR;
+		goto out;
+	}
+
 	if (ubbd_dev->status == UBBD_DEV_USTATUS_ERROR)
 		goto out;
 

lib/ubbd_devs/ubbd_file_dev.c

@@ -24,7 +24,7 @@ static struct ubbd_device *file_dev_create(struct __ubbd_dev_info *info)
 	return ubbd_dev;
 }
 
-static int file_dev_init(struct ubbd_device *ubbd_dev)
+static int file_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	struct ubbd_file_device *file_dev = FILE_DEV(ubbd_dev);
 	int ret;

lib/ubbd_devs/ubbd_mem_dev.c

@@ -22,7 +22,7 @@ static struct ubbd_device *mem_dev_create(struct __ubbd_dev_info *info)
 	return ubbd_dev;
 }
 
-static int mem_dev_init(struct ubbd_device *ubbd_dev)
+static int mem_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	if (ubbd_dev->dev_size > ((uint64_t)UBBD_MEM_BLK_SIZE * UBBD_MEM_BLK_COUNT)) {
 		ubbd_err("dev size for mem type is too large: %lu (max %lu)\n",

lib/ubbd_devs/ubbd_null_dev.c

@@ -22,7 +22,7 @@ static struct ubbd_device *null_dev_create(struct __ubbd_dev_info *info)
 	return ubbd_dev;
 }
 
-static int null_dev_init(struct ubbd_device *ubbd_dev)
+static int null_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	ubbd_dev->dev_features.write_cache = false;
 	ubbd_dev->dev_features.fua = false;

lib/ubbd_devs/ubbd_rbd_dev.c

@@ -1,8 +1,11 @@
+#define _GNU_SOURCE
 #include <rados/librados.h>
 #include <pthread.h>
 
 #include "ubbd_dev.h"
 #include "ubbd_uio.h"
+#include "ubbd_netlink.h"
+#include "utils.h"
 
 // rbd ops
 #define RBD_DEV(ubbd_dev) ((struct ubbd_rbd_device *)container_of(ubbd_dev, struct ubbd_rbd_device, ubbd_dev))
@@ -37,14 +40,42 @@ static struct ubbd_device *rbd_dev_create(struct __ubbd_dev_info *info)
 	strcpy(rbd_conn->user_name, info->rbd.user_name);
 	strcpy(rbd_conn->cluster_name, info->rbd.cluster_name);
 	rbd_conn->io_timeout = info->io_timeout;
+	rbd_conn->update_handle = 0;
 
 	return ubbd_dev;
 }
 
-static int rbd_dev_init(struct ubbd_device *ubbd_dev)
+static void rbd_dev_update_cb(void *arg)
+{
+	struct ubbd_rbd_device *rbd_b = (struct ubbd_rbd_device *)arg;
+	struct ubbd_rbd_conn *rbd_conn = &rbd_b->rbd_conn;
+	struct ubbd_device *ubbd_dev = &rbd_b->ubbd_dev;
+	uint64_t dev_size;
+	int ret;
+
+	ret = ubbd_rbd_get_size(rbd_conn, &dev_size);
+	if (ret < 0) {
+		ubbd_err("failed to get size of ubbd in update watcher.\n");
+		return;
+	}
+
+	if (dev_size != ubbd_dev->dev_size) {
+		ret = ubbd_nl_req_config(ubbd_dev, -1, dev_size, NULL);
+		if (ret) {
+			ubbd_err("failed to send netlink request to resize.\n");
+			return;
+		}
+	}
+
+	ubbd_dev->dev_size = dev_size;
+}
+
+static int rbd_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	struct ubbd_rbd_device *rbd_dev = RBD_DEV(ubbd_dev);
 	struct ubbd_rbd_conn *rbd_conn = &rbd_dev->rbd_conn;
+	char *dev_path;
+	uint64_t dev_size;
         int ret;
 
 	ret = ubbd_rbd_conn_open(rbd_conn);
@@ -61,6 +92,37 @@ static int rbd_dev_init(struct ubbd_device *ubbd_dev)
                 ubbd_dev_info(ubbd_dev, "\nimage get size: %lu.\n", ubbd_dev->dev_size);
         }
 
+	/* check the dev_size for real device */
+	if (reopen) {
+		if (asprintf(&dev_path, "/dev/ubbd%d", ubbd_dev->dev_id) == -1) {
+			ubbd_err("cant init dev path\n");
+			goto close_rbd;
+		}
+
+
+		ret = ubbd_util_get_file_size(dev_path, &dev_size);
+		free(dev_path);
+		if (ret) {
+			ubbd_err("failed to get dev size\n");
+			goto close_rbd;
+		}
+
+		if (dev_size != ubbd_dev->dev_size) {
+			ret = ubbd_nl_req_config(ubbd_dev, -1, ubbd_dev->dev_size, NULL);
+			if (ret) {
+				ubbd_err("failed to send netlink request to update dev_size.\n");
+				goto close_rbd;
+			}
+		}
+	}
+
+	ret = rbd_update_watch(rbd_conn->image, &rbd_conn->update_handle,
+			rbd_dev_update_cb, rbd_dev);
+	if (ret) {
+		ubbd_err("failed to register rbd update watcher:%d\n", ret);
+		goto close_rbd;
+	}
+
 	ubbd_dev->dev_features.write_cache = false;
 	ubbd_dev->dev_features.fua = false;
 	ubbd_dev->dev_features.discard = true;
@@ -73,7 +135,8 @@ static int rbd_dev_init(struct ubbd_device *ubbd_dev)
 		ubbd_dev->dev_features.read_only = true;
 		ubbd_dev->dev_info.read_only = true;
 	}
-	ret = 0;
+
+	return 0;
 
 close_rbd:
 	ubbd_rbd_conn_close(rbd_conn);
@@ -84,7 +147,12 @@ static int rbd_dev_init(struct ubbd_device *ubbd_dev)
 static void rbd_dev_release(struct ubbd_device *ubbd_dev)
 {
 	struct ubbd_rbd_device *rbd_dev = RBD_DEV(ubbd_dev);
+	struct ubbd_rbd_conn *rbd_conn = &rbd_dev->rbd_conn;
 
+	if (rbd_conn->update_handle) {
+		rbd_update_unwatch(rbd_conn->image, rbd_conn->update_handle);
+	}
+	ubbd_rbd_conn_close(rbd_conn);
 	free(rbd_dev);
 }
 

lib/ubbd_devs/ubbd_s3_dev.c

@@ -22,7 +22,7 @@ static struct ubbd_device *s3_dev_create(struct __ubbd_dev_info *info)
 	return ubbd_dev;
 }
 
-static int s3_dev_init(struct ubbd_device *ubbd_dev)
+static int s3_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	ubbd_dev->dev_features.write_cache = false;
 	ubbd_dev->dev_features.fua = false;

lib/ubbd_devs/ubbd_ssh_dev.c

@@ -22,7 +22,7 @@ static struct ubbd_device *ssh_dev_create(struct __ubbd_dev_info *info)
 	return ubbd_dev;
 }
 
-static int ssh_dev_init(struct ubbd_device *ubbd_dev)
+static int ssh_dev_init(struct ubbd_device *ubbd_dev, bool reopen)
 {
 	ubbd_dev->dev_features.write_cache = false;
 	ubbd_dev->dev_features.fua = false;

lib/ubbd_netlink.c

@@ -172,9 +172,18 @@ static int send_netlink_config(struct ubbd_nl_req *req)
 		goto free_msg;
 	}
 
-	ret = nla_put_u32(msg, UBBD_DEV_OPTS_DP_RESERVE, req->req_opts.config_opts.data_pages_reserve_percnt);
-	if (ret < 0)
-		goto free_msg;
+	if (req->req_opts.config_opts.data_pages_reserve_percnt >= 0) {
+		ret = nla_put_u32(msg, UBBD_DEV_OPTS_DP_RESERVE,
+				req->req_opts.config_opts.data_pages_reserve_percnt);
+		if (ret < 0)
+			goto free_msg;
+	}
+
+	if (req->req_opts.config_opts.dev_size > 0) {
+		ret = nla_put_u64(msg, UBBD_DEV_OPTS_DEV_SIZE, req->req_opts.config_opts.dev_size);
+		if (ret < 0)
+			goto free_msg;
+	}
 
 	nla_nest_end(msg, sock_attr);
 
@@ -730,14 +739,16 @@ int ubbd_nl_req_remove_dev(struct ubbd_device *ubbd_dev, struct context *ctx)
 	return ubbd_nl_queue_req(ubbd_dev, req);
 }
 
-int ubbd_nl_req_config(struct ubbd_device *ubbd_dev, int data_pages_reserve_percnt, struct context *ctx)
+int ubbd_nl_req_config(struct ubbd_device *ubbd_dev, int data_pages_reserve_percnt,
+		uint64_t dev_size, struct context *ctx)
 {
 	struct ubbd_nl_req *req = nl_req_alloc();
 
 	INIT_LIST_HEAD(&req->node);
 	req->type = UBBD_NL_REQ_CONFIG;
 	req->ubbd_dev = ubbd_dev;
 	req->req_opts.config_opts.data_pages_reserve_percnt = data_pages_reserve_percnt;
+	req->req_opts.config_opts.dev_size = dev_size;
 	req->ctx = ctx;
 
 	return ubbd_nl_queue_req(ubbd_dev, req);

lib/utils.c

@@ -42,7 +42,7 @@ int ubbd_util_get_file_size(const char *filepath, uint64_t *file_size)
 	int fd;
 	off_t len;
 
-	fd = open(filepath, O_RDWR | O_DIRECT);
+	fd = open(filepath, O_RDONLY);
 	if (fd < 0) {
 		ubbd_err("failed to open filepath: %s: %d\n", filepath, fd);
 		return fd;

ubbdadm/main.c

@@ -313,7 +313,7 @@ int main(int argc, char **argv)
 	int ch, longindex;
 	char *command;
 	int ubbdid;
-	int data_pages_reserve_percnt;
+	int data_pages_reserve_percnt = -1;
 	bool force = false;
 	int ret = 0;
 	bool detach = false;