dog: a new command "node format"

Currently sheepdog lets all nodes have a single store format. This
constraint isn't reasonable. In addition, it is harmful for rolling
updating store format. The rolling update is useful for long living
clusters.

This patch adds a new dog command "dog node format". Example usage:

$ mkdir <sheepdog directory>
$ dog node format <sheepdog directory> <store format (e.g. tree)>

If sheep is launched after the above commands, it prioritizes store
format specified in the node format command than the one specified in
the cluster format command.

Signed-off-by: Hitoshi Mitake <mitake.hitoshi@lab.ntt.co.jp>

Conflicts:
	dog/dog.h
	dog/node.c
	sheep/ops.c

dog/dog.h

@@ -17,6 +17,9 @@
 #include <string.h>
 #include <unistd.h>
 #include <sys/ioctl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
 
 #include "sheepdog_proto.h"
 #include "sheep.h"

dog/node.c

@@ -625,6 +625,68 @@ static int node_log(int argc, char **argv)
 	return do_generic_subcommand(node_log_cmd, argc, argv);
 }
 
+
+static int node_format(int argc, char **argv)
+{
+	char *dir = argv[optind++], *store_name;
+	char config_path[PATH_MAX];
+	struct stat st;
+	int ret, fd;
+	struct node_info info;
+
+	/*
+	 * TODO: for more safe operation, this command should acquire lock which
+	 * is shared with sheep process
+	 */
+
+	if (optind == argc) {
+		sd_err("specify store format");
+		return EXIT_USAGE;
+	}
+	store_name = argv[optind];
+
+	if (strcmp(store_name, "plain") && strcmp(store_name, "tree")) {
+		/*
+		 * FIXME: store names should be macro defined in somewhere
+		 * suitable
+		 */
+		sd_err("expected store format: plain, tree");
+		return EXIT_SYSFAIL;
+	}
+
+	memset(config_path, 0, sizeof(config_path));
+	snprintf(config_path, PATH_MAX, "%s/" NODE_CONFIG_PATH, dir);
+
+	ret = stat(config_path, &st);
+	if (ret < 0) {
+		if (errno != ENOENT) {
+			sd_err("failed to check existence of config (%s): %m",
+			       config_path);
+			return EXIT_SYSFAIL;
+		}
+	} else {
+		/* TODO: dialogue and force option? */
+		sd_err("config file already exists: %s", config_path);
+		return EXIT_SYSFAIL;
+	}
+
+	fd = open(config_path, O_CREAT | O_WRONLY, S_IRUSR | S_IWUSR);
+	if (fd < 0) {
+		sd_err("failed to create a config file (%s): %m", config_path);
+		return EXIT_SYSFAIL;
+	}
+
+	memset(&info, 0, sizeof(info));
+	memcpy(info.store, store_name, STORE_LEN);
+	ret = xwrite(fd, &info, sizeof(info));
+	if (ret != sizeof(info)) {
+		sd_err("failed to write a config file (%s): %m", config_path);
+		return EXIT_SYSFAIL;
+	}
+
+	return 0;
+}
+
 static struct subcommand node_cmd[] = {
 	{"kill", "<node id>", "aprhlT", "kill node", NULL,
 	 CMD_NEED_NODELIST, node_kill, node_options},
@@ -640,6 +702,9 @@ static struct subcommand node_cmd[] = {
 	 0, node_stat, node_options},
 	{"log", NULL, "aphT", "show or set log level of the node", node_log_cmd,
 	 CMD_NEED_ARG, node_log},
+	{"format", "<directory of sheep> <a name of store format>",
+	 "aphT", "initialize store format of the node", NULL, CMD_NEED_ARG,
+	 node_format},
 	{NULL,},
 };
 

include/internal_proto.h

@@ -223,6 +223,18 @@ struct oid_entry {
 	int last;             /* idx to the last element of the oid array */
 };
 
+/*
+ * node_info: node specific configuration
+ * This is a config data initialized with "dog node format".
+ * Currently, it is very simple so doesn't have different expressions
+ * like cluster_info and sheepdog_config.
+ */
+struct node_info {
+	uint8_t store[STORE_LEN];
+};
+
+#define NODE_CONFIG_PATH "node_config"
+
 /*
  * A joining sheep multicasts the local cluster info.  Then, the existing nodes
  * reply the latest cluster info which is unique among all of the nodes.

sheep/config.c

@@ -13,7 +13,7 @@
 
 static struct sheepdog_config config;
 
-char *config_path;
+char *config_path, *node_config_path;
 
 #define CONFIG_PATH "/config"
 
@@ -57,6 +57,32 @@ static int get_cluster_config(struct cluster_info *cinfo)
 	return SD_RES_SUCCESS;
 }
 
+int init_node_config_file(void)
+{
+	int fd, ret;
+	struct stat st;
+
+	ret = stat(node_config_path, &st);
+	if (ret < 0)
+		/* this node doesn't have a node config, do nothing */
+		return 0;
+
+	fd = open(node_config_path, O_RDONLY);
+	if (fd < 0) {
+		sd_err("failed to open node config (%s): %m", node_config_path);
+		return -1;
+	}
+
+	ret = xread(fd, &sys->ninfo, sizeof(sys->ninfo));
+	if (ret != sizeof(sys->ninfo)) {
+		sd_err("failed to read node config (%s): %m", node_config_path);
+		return -1;
+	}
+
+	close(fd);
+	return 0;
+}
+
 int init_config_file(void)
 {
 	int fd, ret = 0;
@@ -135,6 +161,10 @@ void init_config_path(const char *base_path)
 
 	config_path = xzalloc(len);
 	snprintf(config_path, len, "%s" CONFIG_PATH, base_path);
+
+	len = strlen(base_path) + strlen(NODE_CONFIG_PATH) + 1;
+	node_config_path = xzalloc(len);
+	snprintf(node_config_path, len, "%s" NODE_CONFIG_PATH, base_path);
 }
 
 int set_cluster_config(const struct cluster_info *cinfo)

sheep/ops.c

@@ -259,7 +259,11 @@ static int cluster_make_fs(const struct sd_req *req, struct sd_rsp *rsp,
 	struct store_driver *driver;
 	char *store_name = data;
 
-	driver = find_store_driver(data);
+	if (strlen((const char *)sys->ninfo.store))
+		driver = find_store_driver((const char *)sys->ninfo.store);
+	else
+		driver = find_store_driver(data);
+
 	if (!driver)
 		return SD_RES_NO_STORE;
 

sheep/sheep.c

@@ -853,6 +853,10 @@ int main(int argc, char **argv)
 	if (ret)
 		goto cleanup_log;
 
+	ret = init_node_config_file();
+	if (ret)
+		goto cleanup_log;
+
 	ret = init_config_file();
 	if (ret)
 		goto cleanup_log;

sheep/sheep_priv.h

@@ -130,6 +130,7 @@ struct system_info {
 
 	struct sd_node this_node;
 
+	struct node_info ninfo;
 	struct cluster_info cinfo;
 	enum sd_node_status node_status;
 
@@ -437,6 +438,7 @@ int epoch_log_read_remote(uint32_t epoch, struct sd_node *nodes,
 				struct vnode_info *vinfo);
 uint32_t get_latest_epoch(void);
 void init_config_path(const char *base_path);
+int init_node_config_file(void);
 int init_config_file(void);
 int get_obj_list(const struct sd_req *, struct sd_rsp *, void *);
 int objlist_cache_cleanup(uint32_t vid);

sheep/store/common.c

@@ -364,8 +364,12 @@ int init_store_driver(bool is_gateway)
 {
 	char driver_name[STORE_LEN], *p;
 
-	pstrcpy(driver_name, sizeof(driver_name),
-		(char *)sys->cinfo.default_store);
+	if (strlen((const char *)sys->ninfo.store))
+		pstrcpy(driver_name, sizeof(driver_name),
+			(char *)sys->ninfo.store);
+	else
+		pstrcpy(driver_name, sizeof(driver_name),
+			(char *)sys->cinfo.default_store);
 
 	p = memchr(driver_name, '\0', STORE_LEN);
 	if (!p) {