diff --git a/doc/man1/flux-kvs.adoc b/doc/man1/flux-kvs.adoc index aa5f85330ba3..1b30431f71a0 100644 --- a/doc/man1/flux-kvs.adoc +++ b/doc/man1/flux-kvs.adoc @@ -42,13 +42,22 @@ arguments are described below. COMMANDS -------- -*get* 'key' ['key...']:: +*get* [-j|-r|-t] [-a treeobj] 'key' ['key...']:: Retrieve the value stored under 'key'. If nothing has been stored under -'key', display an error message. +'key', display an error message. If no options, value is interpreted +as a NULL-terminated string. If '-j', it is interpreted as encoded JSON. +If '-r', it is interpreted as raw data and is output without formatting. +If '-t', the RFC 11 object is displayed. '-a treeobj' causes the lookup +to be relative to an RFC 11 snapshot reference. -*put* 'key=value' ['key=value...']:: +*put* [-j|-r|-t] [-n] 'key=value' ['key=value...']:: Store 'value' under 'key' and commit it. If it already has a value, -overwrite it. +overwrite it. If no options, value is stored as a NULL-terminated string. +If '-j', it is first encoded as JSON, then stored as a NULL-terminated string. +If '-r', it is stored as raw data with no termination. For raw mode only, +a value of "-" indicates that the value should be read from standard input. +If '-t', value is stored as a RFC 11 object. '-n' prevents the commit +from being merged with with other contemporaneous commits. *ls* [-R] [-d] [-F] [-w COLS] [-1] ['key' ...]:: Display directory referred to by _key_, or "." (root) if unspecified. @@ -58,12 +67,14 @@ Options are roughly equivalent to a subset of ls(1) options. '-w COLS' sets the terminal width in characters. '-1' causes output to be displayed in one column. -*dir* [-R] [-d] ['key']:: +*dir* [-R] [-d] [-w COLS] [-a treeobj] ['key']:: Display all keys and their values under the directory 'key'. If 'key' does not exist or is not a directory, display an error message. If 'key' is not provided, "." (root of the namespace) is assumed. If '-R' is specified, recursively display keys under subdirectories. If '-d' is -specified, do not output key values. +specified, do not output key values. Output is truncated to fit the +terminal width. '-w COLS' sets the terminal width (0=unlimited). +'-a treeobj' causes the lookup to be relative to an RFC 11 snapshot reference. *unlink* [-R] [-f] 'key' ['key...']:: Remove 'key' from the KVS and commit the change. If 'key' represents @@ -75,9 +86,10 @@ Create a new name for 'target', similar to a symbolic link, and commit the change. 'target' does not have to exist. If 'linkname' exists, it is overwritten. -*readlink* 'key' ['key...']:: +*readlink* [-a treeobj] 'key' ['key...']:: Retrieve the key a link refers to rather than its value, as would be -returned by *get*. +returned by *get*. '-a treeobj' causes the lookup to be relative to +an RFC 11 snapshot reference. *mkdir* 'key' ['key...']:: Create an empty directory and commit the change. If 'key' exists, diff --git a/src/cmd/flux-kvs.c b/src/cmd/flux-kvs.c index ba0ad6265623..80313f6be1cd 100644 --- a/src/cmd/flux-kvs.c +++ b/src/cmd/flux-kvs.c @@ -52,10 +52,51 @@ int cmd_move (optparse_t *p, int argc, char **argv); int cmd_dir (optparse_t *p, int argc, char **argv); int cmd_ls (optparse_t *p, int argc, char **argv); -static void dump_kvs_dir (const flux_kvsdir_t *dir, bool Ropt, bool dopt); +static int get_window_width (optparse_t *p, int fd); +static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol, + bool Ropt, bool dopt); #define min(a,b) ((a)<(b)?(a):(b)) +static struct optparse_option readlink_opts[] = { + { .name = "at", .key = 'a', .has_arg = 1, + .usage = "Lookup relative to RFC 11 snapshot reference", + }, + OPTPARSE_TABLE_END +}; + +static struct optparse_option get_opts[] = { + { .name = "json", .key = 'j', .has_arg = 0, + .usage = "Interpret value(s) as encoded JSON", + }, + { .name = "raw", .key = 'r', .has_arg = 0, + .usage = "Interpret value(s) as raw data", + }, + { .name = "treeobj", .key = 't', .has_arg = 0, + .usage = "Show RFC 11 object", + }, + { .name = "at", .key = 'a', .has_arg = 1, + .usage = "Lookup relative to RFC 11 snapshot reference", + }, + OPTPARSE_TABLE_END +}; + +static struct optparse_option put_opts[] = { + { .name = "json", .key = 'j', .has_arg = 0, + .usage = "Store value(s) as encoded JSON", + }, + { .name = "raw", .key = 'r', .has_arg = 0, + .usage = "Store value(s) as-is without adding NULL termination", + }, + { .name = "treeobj", .key = 't', .has_arg = 0, + .usage = "Store RFC 11 object", + }, + { .name = "no-merge", .key = 'n', .has_arg = 0, + .usage = "Set the NO_MERGE flag to ensure commit is standalone", + }, + OPTPARSE_TABLE_END +}; + static struct optparse_option dir_opts[] = { { .name = "recursive", .key = 'R', .has_arg = 0, .usage = "Recursively display keys under subdirectories", @@ -63,6 +104,12 @@ static struct optparse_option dir_opts[] = { { .name = "directory", .key = 'd', .has_arg = 0, .usage = "List directory entries and not values", }, + { .name = "width", .key = 'w', .has_arg = 1, + .usage = "Set output width to COLS. 0 means no limit", + }, + { .name = "at", .key = 'a', .has_arg = 1, + .usage = "Lookup relative to RFC 11 snapshot reference", + }, OPTPARSE_TABLE_END }; @@ -120,21 +167,21 @@ static struct optparse_option unlink_opts[] = { static struct optparse_subcommand subcommands[] = { { "get", - "key [key...]", + "[-j|-r|-t] [-a treeobj] key [key...]", "Get value stored under key", cmd_get, 0, - NULL + get_opts }, { "put", - "key=value [key=value...]", + "[-j|-r|-t] [-n] key=value [key=value...]", "Store value under key", cmd_put, 0, - NULL + put_opts }, { "dir", - "[-R] [-d] [key]", + "[-R] [-d] [-w COLS] [-a treeobj] [key]", "Display all keys under directory", cmd_dir, 0, @@ -162,11 +209,11 @@ static struct optparse_subcommand subcommands[] = { NULL }, { "readlink", - "key [key...]", + "[-a treeobj] key [key...]", "Retrieve the key a link refers to", cmd_readlink, 0, - NULL + readlink_opts }, { "mkdir", "key [key...]", @@ -284,37 +331,86 @@ int main (int argc, char *argv[]) return (exitval); } -static void output_key_json_object (const char *key, json_t *o) +static void kv_printf (const char *key, int maxcol, const char *fmt, ...) +{ + va_list ap; + int rc; + char *val, *kv, *p; + bool overflow = false; + + if (maxcol != 0 && maxcol <= 3) + maxcol = 0; + + va_start (ap, fmt); + rc = vasprintf (&val, fmt, ap); + va_end (ap); + + if (rc < 0) + log_err_exit ("%s", __FUNCTION__); + + if (asprintf (&kv, "%s%s%s", key ? key : "", + key ? " = " : "", + val) < 0) + log_err_exit ("%s", __FUNCTION__); + + /* There will be no truncation of output if maxcol = 0. + * If truncation is enabled, ensure that at most maxcol columns are used. + * Truncate earlier if value contains a convenient newline break. + * Finally, truncate on first non-printable character. + */ + if (maxcol != 0) { + if (strlen (kv) > maxcol - 3) { + kv[maxcol - 3] = '\0'; + overflow = true; + } + if ((p = strchr (kv, '\n'))) { + *p = '\0'; + overflow = true; + } + for (p = kv; *p != '\0'; p++) { + if (!isprint (*p)) { + *p = '\0'; + overflow = true; + break; + } + } + } + printf ("%s%s\n", kv, + overflow ? "..." : ""); + + free (val); + free (kv); +} + +static void output_key_json_object (const char *key, json_t *o, int maxcol) { char *s; - if (key) - printf ("%s = ", key); switch (json_typeof (o)) { case JSON_NULL: - printf ("nil\n"); + kv_printf (key, maxcol, "nil"); break; case JSON_TRUE: - printf ("true\n"); + kv_printf (key, maxcol, "true"); break; case JSON_FALSE: - printf ("false\n"); + kv_printf (key, maxcol, "false"); break; case JSON_REAL: - printf ("%f\n", json_real_value (o)); + kv_printf (key, maxcol, "%f", json_real_value (o)); break; case JSON_INTEGER: - printf ("%lld\n", (long long)json_integer_value (o)); + kv_printf (key, maxcol, "%lld", (long long)json_integer_value (o)); break; case JSON_STRING: - printf ("%s\n", json_string_value (o)); + kv_printf (key, maxcol, "%s", json_string_value (o)); break; case JSON_ARRAY: case JSON_OBJECT: default: if (!(s = json_dumps (o, JSON_SORT_KEYS))) log_msg_exit ("json_dumps failed"); - printf ("%s\n", s); + kv_printf (key, maxcol, "%s", s); free (s); break; } @@ -332,14 +428,13 @@ static void output_key_json_str (const char *key, if (!(o = json_loads (json_str, JSON_DECODE_ANY, &error))) log_msg_exit ("%s: %s (line %d column %d)", arg, error.text, error.line, error.column); - output_key_json_object (key, o); + output_key_json_object (key, o, 0); json_decref (o); } int cmd_get (optparse_t *p, int argc, char **argv) { flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle"); - const char *key, *json_str; flux_future_t *f; int optindex, i; @@ -349,11 +444,48 @@ int cmd_get (optparse_t *p, int argc, char **argv) exit (1); } for (i = optindex; i < argc; i++) { - key = argv[i]; - if (!(f = flux_kvs_lookup (h, 0, key)) - || flux_kvs_lookup_get (f, &json_str) < 0) - log_err_exit ("%s", key); - output_key_json_str (NULL, json_str, key); + const char *key = argv[i]; + int flags = 0; + if (optparse_hasopt (p, "treeobj")) + flags |= FLUX_KVS_TREEOBJ; + if (optparse_hasopt (p, "at")) { + const char *reference = optparse_get_str (p, "at", ""); + if (!(f = flux_kvs_lookupat (h, flags, key, reference))) + log_err_exit ("%s", key); + } + else { + if (!(f = flux_kvs_lookup (h, flags, key))) + log_err_exit ("%s", key); + } + if (optparse_hasopt (p, "treeobj")) { + const char *treeobj; + if (flux_kvs_lookup_get_treeobj (f, &treeobj) < 0) + log_err_exit ("%s", key); + printf ("%s\n", treeobj); + } + else if (optparse_hasopt (p, "json")) { + const char *json_str; + if (flux_kvs_lookup_get (f, &json_str) < 0) + log_err_exit ("%s", key); + if (!json_str) + log_msg_exit ("%s: zero-length value", key); + output_key_json_str (NULL, json_str, key); + } + else if (optparse_hasopt (p, "raw")) { + const void *data; + int len; + if (flux_kvs_lookup_get_raw (f, &data, &len) < 0) + log_err_exit ("%s", key); + if (write_all (STDOUT_FILENO, data, len) < 0) + log_err_exit ("%s", key); + } + else { + const char *value; + if (flux_kvs_lookup_get (f, &value) < 0) + log_err_exit ("%s", key); + if (value) + printf ("%s\n", value); + } flux_future_destroy (f); } return (0); @@ -365,12 +497,15 @@ int cmd_put (optparse_t *p, int argc, char **argv) int optindex, i; flux_future_t *f; flux_kvs_txn_t *txn; + int commit_flags = 0; optindex = optparse_option_index (p); if ((optindex - argc) == 0) { optparse_print_usage (p); exit (1); } + if (optparse_hasopt (p, "no-merge")) + commit_flags |= FLUX_KVS_NO_MERGE; if (!(txn = flux_kvs_txn_create ())) log_err_exit ("flux_kvs_txn_create"); for (i = optindex; i < argc; i++) { @@ -380,19 +515,43 @@ int cmd_put (optparse_t *p, int argc, char **argv) log_msg_exit ("put: you must specify a value as key=value"); *val++ = '\0'; - json_t *obj; - if ((obj = json_loads (val, JSON_DECODE_ANY, NULL))) { - if (flux_kvs_txn_put (txn, 0, key, val) < 0) + if (optparse_hasopt (p, "treeobj")) { + if (flux_kvs_txn_put_treeobj (txn, 0, key, val) < 0) + log_err_exit ("%s", key); + } + else if (optparse_hasopt (p, "json")) { + json_t *obj; + if ((obj = json_loads (val, JSON_DECODE_ANY, NULL))) { + if (flux_kvs_txn_put (txn, 0, key, val) < 0) + log_err_exit ("%s", key); + json_decref (obj); + } + else { // encode as JSON string if not already valid encoded JSON + if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0) + log_err_exit ("%s", key); + } + } + else if (optparse_hasopt (p, "raw")) { + int len; + uint8_t *buf = NULL; + if (!strcmp (val, "-")) { // special handling for "--raw key=-" + if ((len = read_all (STDIN_FILENO, &buf)) < 0) + log_err_exit ("stdin"); + val = (char *)buf; + } else + len = strlen (val); + if (flux_kvs_txn_put_raw (txn, 0, key, val, len) < 0) log_err_exit ("%s", key); - json_decref (obj); + free (buf); } - else { // encode as JSON string if not already valid encoded JSON - if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0) + else { + if (flux_kvs_txn_put (txn, 0, key, val) < 0) log_err_exit ("%s", key); } free (key); } - if (!(f = flux_kvs_commit (h, 0, txn)) || flux_future_get (f, NULL) < 0) + if (!(f = flux_kvs_commit (h, commit_flags, txn)) + || flux_future_get (f, NULL) < 0) log_err_exit ("flux_kvs_commit"); flux_future_destroy (f); flux_kvs_txn_destroy (txn); @@ -509,11 +668,18 @@ int cmd_readlink (optparse_t *p, int argc, char **argv) } for (i = optindex; i < argc; i++) { - if (!(f = flux_kvs_lookup (h, FLUX_KVS_READLINK, argv[i])) - || flux_kvs_lookup_get_symlink (f, &target) < 0) + if (optparse_hasopt (p, "at")) { + const char *ref = optparse_get_str (p, "at", ""); + if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READLINK, argv[i], ref))) + log_err_exit ("%s", argv[i]); + } + else { + if (!(f = flux_kvs_lookup (h, FLUX_KVS_READLINK, argv[i]))) + log_err_exit ("%s", argv[i]); + } + if (flux_kvs_lookup_get_symlink (f, &target) < 0) log_err_exit ("%s", argv[i]); - else - printf ("%s\n", target); + printf ("%s\n", target); flux_future_destroy (f); } return (0); @@ -599,7 +765,7 @@ static void watch_dump_kvsdir (flux_kvsdir_t *dir, bool Ropt, bool dopt, return; } - dump_kvs_dir (dir, Ropt, dopt); + dump_kvs_dir (dir, 0, Ropt, dopt); printf ("%s\n", WATCH_DIR_SEPARATOR); fflush (stdout); } @@ -754,23 +920,24 @@ int cmd_dropcache (optparse_t *p, int argc, char **argv) return (0); } -static void dump_kvs_val (const char *key, const char *json_str) +static void dump_kvs_val (const char *key, int maxcol, const char *value) { json_t *o; - json_error_t error; - if (!json_str) - json_str = "null"; - if (!(o = json_loads (json_str, JSON_DECODE_ANY, &error))) { - printf ("%s: %s (line %d column %d)\n", - key, error.text, error.line, error.column); - return; + if (!value) { + kv_printf (key, maxcol, ""); + } + else if ((o = json_loads (value, JSON_DECODE_ANY, NULL))) { + output_key_json_object (key, o, maxcol); + json_decref (o); + } + else { + kv_printf (key, maxcol, value); } - output_key_json_object (key, o); - json_decref (o); } -static void dump_kvs_dir (const flux_kvsdir_t *dir, bool Ropt, bool dopt) +static void dump_kvs_dir (const flux_kvsdir_t *dir, int maxcol, + bool Ropt, bool dopt) { const char *rootref = flux_kvsdir_rootref (dir); flux_t *h = flux_kvsdir_handle (dir); @@ -796,17 +963,23 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, bool Ropt, bool dopt) if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, key, rootref)) || flux_kvs_lookup_get_dir (f, &ndir) < 0) log_err_exit ("%s", key); - dump_kvs_dir (ndir, Ropt, dopt); + dump_kvs_dir (ndir, maxcol, Ropt, dopt); flux_future_destroy (f); } else printf ("%s.\n", key); } else { if (!dopt) { - const char *json_str; - if (!(f = flux_kvs_lookupat (h, 0, key, rootref)) - || flux_kvs_lookup_get (f, &json_str) < 0) + const char *value; + const void *buf; + int len; + if (!(f = flux_kvs_lookupat (h, 0, key, rootref))) + log_err_exit ("%s", key); + if (flux_kvs_lookup_get (f, &value) == 0) // null terminated + dump_kvs_val (key, maxcol, value); + else if (flux_kvs_lookup_get_raw (f, &buf, &len) == 0) + kv_printf (key, maxcol, "%.*s", len, buf); + else log_err_exit ("%s", key); - dump_kvs_val (key, json_str); flux_future_destroy (f); } else @@ -820,6 +993,7 @@ static void dump_kvs_dir (const flux_kvsdir_t *dir, bool Ropt, bool dopt) int cmd_dir (optparse_t *p, int argc, char **argv) { flux_t *h = (flux_t *)optparse_get_data (p, "flux_handle"); + int maxcol = get_window_width (p, STDOUT_FILENO); bool Ropt; bool dopt; const char *key; @@ -837,10 +1011,18 @@ int cmd_dir (optparse_t *p, int argc, char **argv) else log_msg_exit ("dir: specify zero or one directory"); - if (!(f = flux_kvs_lookup (h, FLUX_KVS_READDIR, key)) - || flux_kvs_lookup_get_dir (f, &dir) < 0) + if (optparse_hasopt (p, "at")) { + const char *reference = optparse_get_str (p, "at", ""); + if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, key, reference))) + log_err_exit ("%s", key); + } + else { + if (!(f = flux_kvs_lookup (h, FLUX_KVS_READDIR, key))) + log_err_exit ("%s", key); + } + if (flux_kvs_lookup_get_dir (f, &dir) < 0) log_err_exit ("%s", key); - dump_kvs_dir (dir, Ropt, dopt); + dump_kvs_dir (dir, maxcol, Ropt, dopt); flux_future_destroy (f); return (0); } @@ -891,8 +1073,6 @@ static int get_window_width (optparse_t *p, int fd) int width; const char *s; - if (optparse_hasopt (p, "1")) - return 1; if ((width = optparse_get_int (p, "width", -1)) >= 0) return width; if (ioctl (fd, TIOCGWINSZ, &w) == 0) @@ -1124,6 +1304,9 @@ int cmd_ls (optparse_t *p, int argc, char **argv) bool print_vspace = false; // print vertical space before label int rc = 0; + if (optparse_hasopt (p, "1")) + win_width = 1; + if (!(dirs = zlist_new ()) || !(singles = zlist_new ())) log_err_exit ("zlist_new"); diff --git a/t/issues/t0441-kvs-put-get.sh b/t/issues/t0441-kvs-put-get.sh index c1e571bd645a..f5b844eb7b72 100755 --- a/t/issues/t0441-kvs-put-get.sh +++ b/t/issues/t0441-kvs-put-get.sh @@ -3,8 +3,8 @@ TEST=issue441 -flux kvs put ${TEST}.x=foo +flux kvs put --json ${TEST}.x=foo -flux kvs get ${TEST}.x.y && test $? -eq 1 +flux kvs get --json ${TEST}.x.y && test $? -eq 1 -flux kvs get ${TEST}.x # fails if broker died +flux kvs get --json ${TEST}.x # fails if broker died diff --git a/t/issues/t0821-kvs-segfault.sh b/t/issues/t0821-kvs-segfault.sh index 5f00d65799f7..94c617ef98b4 100755 --- a/t/issues/t0821-kvs-segfault.sh +++ b/t/issues/t0821-kvs-segfault.sh @@ -2,6 +2,6 @@ # kvs put test="large string", get test.x fails without panic TEST=issue0821 -flux kvs put ${TEST}="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" -flux kvs get ${TEST}.x && test $? -eq 1 -flux kvs get ${TEST} # fails if broker died +flux kvs put --json ${TEST}="xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" +flux kvs get --json ${TEST}.x && test $? -eq 1 +flux kvs get --json ${TEST} # fails if broker died diff --git a/t/kvs/basic.c b/t/kvs/basic.c index c84e1aca0343..8b945fe4e7ee 100644 --- a/t/kvs/basic.c +++ b/t/kvs/basic.c @@ -44,30 +44,12 @@ static const struct option longopts[] = { }; void cmd_type (flux_t *h, int argc, char **argv); -void cmd_put_no_merge (flux_t *h, int argc, char **argv); -void cmd_copy_tokvs (flux_t *h, int argc, char **argv); -void cmd_copy_fromkvs (flux_t *h, int argc, char **argv); -void cmd_dirsize (flux_t *h, int argc, char **argv); -void cmd_get_treeobj (flux_t *h, int argc, char **argv); -void cmd_put_treeobj (flux_t *h, int argc, char **argv); -void cmd_getat (flux_t *h, int argc, char **argv); -void cmd_dirat (flux_t *h, int argc, char **argv); -void cmd_readlinkat (flux_t *h, int argc, char **argv); void usage (void) { fprintf (stderr, "Usage: basic type key\n" -" basic put-no-merge key=val\n" -" basic copy-tokvs key file\n" -" basic copy-fromkvs key file\n" -" basic dirsize key\n" -" basic get-treeobj key\n" -" basic put-treeobj key=treeobj\n" -" basic getat treeobj key\n" -" basic dirat [-r] treeobj [key]\n" -" basic readlinkat treeobj key\n" ); exit (1); } @@ -99,24 +81,6 @@ int main (int argc, char *argv[]) if (!strcmp (cmd, "type")) cmd_type (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "put-no-merge")) - cmd_put_no_merge (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "copy-tokvs")) - cmd_copy_tokvs (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "copy-fromkvs")) - cmd_copy_fromkvs (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "dirsize")) - cmd_dirsize (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "get-treeobj")) - cmd_get_treeobj (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "put-treeobj")) - cmd_put_treeobj (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "getat")) - cmd_getat (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "dirat")) - cmd_dirat (h, argc - optind, argv + optind); - else if (!strcmp (cmd, "readlinkat")) - cmd_readlinkat (h, argc - optind, argv + optind); else usage (); @@ -170,315 +134,6 @@ void cmd_type (flux_t *h, int argc, char **argv) flux_future_destroy (f); } -static void output_key_json_object (const char *key, json_t *o) -{ - char *s; - if (key) - printf ("%s = ", key); - - switch (json_typeof (o)) { - case JSON_NULL: - printf ("nil\n"); - break; - case JSON_TRUE: - printf ("true\n"); - break; - case JSON_FALSE: - printf ("false\n"); - break; - case JSON_REAL: - printf ("%f\n", json_real_value (o)); - break; - case JSON_INTEGER: - printf ("%lld\n", (long long)json_integer_value (o)); - break; - case JSON_STRING: - printf ("%s\n", json_string_value (o)); - break; - case JSON_ARRAY: - case JSON_OBJECT: - default: - if (!(s = json_dumps (o, JSON_SORT_KEYS))) - log_msg_exit ("json_dumps failed"); - printf ("%s\n", s); - free (s); - break; - } -} - -static void output_key_json_str (const char *key, - const char *json_str, - const char *arg) -{ - json_t *o; - json_error_t error; - - if (!json_str) - json_str = "null"; - if (!(o = json_loads (json_str, JSON_DECODE_ANY, &error))) - log_msg_exit ("%s: %s (line %d column %d)", - arg, error.text, error.line, error.column); - output_key_json_object (key, o); - json_decref (o); -} - -void cmd_put_no_merge (flux_t *h, int argc, char **argv) -{ - flux_kvs_txn_t *txn; - flux_future_t *f; - - if (argc == 0) - log_msg_exit ("put: specify one key=value pair"); - char *key = xstrdup (argv[0]); - char *val = strchr (key, '='); - if (!val) - log_msg_exit ("put: you must specify a value as key=value"); - *val++ = '\0'; - - if (!(txn = flux_kvs_txn_create ())) - log_err_exit( "flux_kvs_txn_create"); - if (flux_kvs_txn_put (txn, 0, key, val) < 0) { - if (errno != EINVAL) - log_err_exit ("%s", key); - if (flux_kvs_txn_pack (txn, 0, key, "s", val) < 0) - log_err_exit ("%s", key); - } - free (key); - if (!(f = flux_kvs_commit (h, FLUX_KVS_NO_MERGE, txn)) - || flux_future_get (f, NULL) < 0) - log_err_exit ("flux_kvs_commit"); - flux_future_destroy (f); - flux_kvs_txn_destroy (txn); -} - -void cmd_copy_tokvs (flux_t *h, int argc, char **argv) -{ - char *file, *key; - int fd, len; - uint8_t *buf = NULL; - flux_kvs_txn_t *txn; - flux_future_t *f; - - if (argc != 2) - log_msg_exit ("copy-tokvs: specify key and filename"); - key = argv[0]; - file = argv[1]; - if (!strcmp (file, "-")) { - if ((len = read_all (STDIN_FILENO, &buf)) < 0) - log_err_exit ("stdin"); - } else { - if ((fd = open (file, O_RDONLY)) < 0) - log_err_exit ("%s", file); - if ((len = read_all (fd, &buf)) < 0) - log_err_exit ("%s", file); - (void)close (fd); - } - if (!(txn = flux_kvs_txn_create ())) - log_err_exit ("flux_kvs_txn_create"); - if (flux_kvs_txn_put_raw (txn, 0, key, buf, len) < 0) - log_err_exit ("flux_kvs_txn_put_raw"); - if (!(f = flux_kvs_commit (h, 0, txn)) || flux_future_get (f, NULL) < 0) - log_err_exit ("flux_kvs_commit"); - flux_kvs_txn_destroy (txn); - free (buf); -} - -void cmd_copy_fromkvs (flux_t *h, int argc, char **argv) -{ - char *file, *key; - int fd, len; - const uint8_t *buf = NULL; - flux_future_t *f; - - if (argc != 2) - log_msg_exit ("copy-fromkvs: specify key and filename"); - key = argv[0]; - file = argv[1]; - if (!(f = flux_kvs_lookup (h, 0, key))) - log_err_exit ("flux_kvs_lookup"); - if (flux_kvs_lookup_get_raw (f, (const void **)&buf, &len) < 0) - log_err_exit ("%s", key); - if (!strcmp (file, "-")) { - if (write_all (STDOUT_FILENO, buf, len) < 0) - log_err_exit ("stdout"); - } else { - if ((fd = creat (file, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)) < 0) - log_err_exit ("%s", file); - if (write_all (fd, buf, len) < 0) - log_err_exit ("%s", file); - if (close (fd) < 0) - log_err_exit ("%s", file); - } - flux_future_destroy (f); -} - -static void dump_kvs_val (const char *key, const char *json_str) -{ - json_t *o; - json_error_t error; - - if (!json_str) - json_str = "null"; - if (!(o = json_loads (json_str, JSON_DECODE_ANY, &error))) { - printf ("%s: %s (line %d column %d)\n", - key, error.text, error.line, error.column); - return; - } - output_key_json_object (key, o); - json_decref (o); -} - -static void dump_kvs_dir (const flux_kvsdir_t *dir, bool ropt) -{ - flux_future_t *f; - flux_kvsitr_t *itr; - const char *name; - flux_t *h = flux_kvsdir_handle (dir); - const char *rootref = flux_kvsdir_rootref (dir); - char *key; - - itr = flux_kvsitr_create (dir); - while ((name = flux_kvsitr_next (itr))) { - key = flux_kvsdir_key_at (dir, name); - if (flux_kvsdir_issymlink (dir, name)) { - const char *link; - if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READLINK, key, rootref)) - || flux_kvs_lookup_get_symlink (f, &link) < 0) - log_err_exit ("%s", key); - printf ("%s -> %s\n", key, link); - flux_future_destroy (f); - - } else if (flux_kvsdir_isdir (dir, name)) { - if (ropt) { - const flux_kvsdir_t *ndir; - if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, key, rootref)) - || flux_kvs_lookup_get_dir (f, &ndir) < 0) - log_err_exit ("%s", key); - dump_kvs_dir (ndir, ropt); - flux_future_destroy (f); - } else - printf ("%s.\n", key); - } else { - const char *json_str; - if (!(f = flux_kvs_lookupat (h, 0, key, rootref)) - || flux_kvs_lookup_get (f, &json_str) < 0) - log_err_exit ("%s", key); - dump_kvs_val (key, json_str); - flux_future_destroy (f); - } - free (key); - } - flux_kvsitr_destroy (itr); -} - -void cmd_dirat (flux_t *h, int argc, char **argv) -{ - bool ropt = false; - const flux_kvsdir_t *dir; - flux_future_t *f; - - if (argc > 0) { - while (argc) { - if (!strcmp (argv[0], "-r")) { - ropt = true; - argc--; - argv++; - } - else - break; - } - } - if (argc != 2) - log_msg_exit ("dir: specify treeobj and directory"); - if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READDIR, argv[1], argv[0])) - || flux_kvs_lookup_get_dir (f, &dir) < 0) - log_err_exit ("%s", argv[1]); - dump_kvs_dir (dir, ropt); - flux_future_destroy (f); -} - -void cmd_dirsize (flux_t *h, int argc, char **argv) -{ - flux_future_t *f; - const flux_kvsdir_t *dir = NULL; - - if (argc != 1) - log_msg_exit ("dirsize: specify one directory"); - if (!(f = flux_kvs_lookup (h, FLUX_KVS_READDIR, argv[0]))) - log_err_exit ("flux_kvs_lookup"); - if (flux_kvs_lookup_get_dir (f, &dir) < 0) - log_err_exit ("%s", argv[0]); - printf ("%d\n", flux_kvsdir_get_size (dir)); - flux_future_destroy (f); -} - -void cmd_get_treeobj (flux_t *h, int argc, char **argv) -{ - const char *treeobj; - flux_future_t *f; - - if (argc != 1) - log_msg_exit ("get-treeobj: specify key"); - if (!(f = flux_kvs_lookup (h, FLUX_KVS_TREEOBJ, argv[0])) - || flux_kvs_lookup_get_treeobj (f, &treeobj) < 0) - log_err_exit ("kvs_get_treeobj %s", argv[0]); - printf ("%s\n", treeobj); - flux_future_destroy (f); -} - -void cmd_getat (flux_t *h, int argc, char **argv) -{ - const char *json_str; - flux_future_t *f; - - if (argc != 2) - log_msg_exit ("getat: specify treeobj and key"); - if (!(f = flux_kvs_lookupat (h, 0, argv[1], argv[0])) - || flux_kvs_lookup_get (f, &json_str) < 0) - log_err_exit ("flux_kvs_lookupat %s %s", argv[0], argv[1]); - output_key_json_str (NULL, json_str, argv[1]); - flux_future_destroy (f); -} - -void cmd_put_treeobj (flux_t *h, int argc, char **argv) -{ - flux_future_t *f; - flux_kvs_txn_t *txn; - - if (argc != 1) - log_msg_exit ("put-treeobj: specify key=val"); - char *key = xstrdup (argv[0]); - char *val = strchr (key, '='); - if (!val) - log_msg_exit ("put-treeobj: you must specify a value as key=val"); - *val++ = '\0'; - - if (!(txn = flux_kvs_txn_create ())) - log_err_exit ("flux_kvs_txn_create"); - if (flux_kvs_txn_put_treeobj (txn, 0, key, val) < 0) - log_err_exit ("flux_kvs_txn_put %s=%s", key, val); - if (!(f = flux_kvs_commit (h, 0, txn)) || flux_future_get (f, NULL) < 0) - log_err_exit ("flux_kvs_commit"); - flux_future_destroy (f); - flux_kvs_txn_destroy (txn); - free (key); -} - -void cmd_readlinkat (flux_t *h, int argc, char **argv) -{ - const char *target; - flux_future_t *f; - - if (argc != 2) - log_msg_exit ("readlink: specify treeobj and key"); - if (!(f = flux_kvs_lookupat (h, FLUX_KVS_READLINK, argv[1], argv[0])) - || flux_kvs_lookup_get_symlink (f, &target) < 0) - log_err_exit ("%s", argv[1]); - else - printf ("%s\n", target); - flux_future_destroy (f); -} - /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/t/lua/t1002-kvs.t b/t/lua/t1002-kvs.t index ed2e261b8623..ee573a0aab26 100755 --- a/t/lua/t1002-kvs.t +++ b/t/lua/t1002-kvs.t @@ -196,7 +196,7 @@ type_ok (kw, 'userdata', "f:kvswatcher returns kvswatcher object") kw.testkey = "foo" is (kw.testkey, 'foo', "Can set arbitrary members of kvswatcher object") -os.execute (string.format ("flux kvs put %s=%s", data.key, data.value)) +os.execute (string.format ("flux kvs put --json %s=%s", data.key, data.value)) local to = f:timer { timeout = 1500, @@ -247,7 +247,7 @@ local t, err = f:timer { } -- Excute on rank 3 via flux-exec: -os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put %s=%s", +os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put --json %s=%s", data.key, data.value)) local r, err = f:reactor() isnt (r, -1, "reactor exited normally with ".. (r and r or err)) @@ -255,7 +255,7 @@ is (ncount, 2, "kvswatch callback invoked exactly twice") note ("Ensure kvs watch callback not invoked after kvswatcher removal") ok (kw:remove(), "Can remove kvswatcher without error") -os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put %s=%s", +os.execute (string.format ("sleep 0.25 && flux exec -r 3 flux kvs put --json %s=%s", data.key, 'test3')) local t, err = f:timer { diff --git a/t/t0017-security.t b/t/t0017-security.t index 203072a5d428..a017bc68212b 100755 --- a/t/t0017-security.t +++ b/t/t0017-security.t @@ -279,7 +279,7 @@ test_expect_success 'connector delivers kvs.setroot event to owner connection' ' $SHARNESS_TEST_SRCDIR/scripts/event-trace-bypass.lua \ kvs kvs.test.end \ "flux event pub kvs.test.a; \ - flux kvs put ev9=42; \ + flux kvs put --json ev9=42; \ flux event pub kvs.test.end" >ev9.out && grep -q kvs.setroot ev9.out ' @@ -289,7 +289,7 @@ test_expect_success 'dispatcher delivers kvs.setroot event to owner connection' $SHARNESS_TEST_SRCDIR/scripts/event-trace.lua \ kvs kvs.test.end \ "flux event pub kvs.test.a; \ - flux kvs put ev10=42; \ + flux kvs put --json ev10=42; \ flux event pub kvs.test.end" >ev10.out && grep -q kvs.setroot ev10.out ' @@ -300,7 +300,7 @@ test_expect_success 'connector suppresses kvs.setroot event to guest connection' $SHARNESS_TEST_SRCDIR/scripts/event-trace-bypass.lua \ kvs kvs.test.end \ "flux event pub kvs.test.a; \ - flux kvs put ev11=42; \ + flux kvs put --json ev11=42; \ flux event pub kvs.test.end" >ev11.out && ! grep -q kvs.setroot ev11.out ' diff --git a/t/t1000-kvs.t b/t/t1000-kvs.t index 7708b06b55e0..cd7bafed28cc 100755 --- a/t/t1000-kvs.t +++ b/t/t1000-kvs.t @@ -25,9 +25,9 @@ SUBDIR1=test.a.b.d SUBDIR2=test.a.b.e test_kvs_key() { - flux kvs get "$1" >output + flux kvs get --json "$1" >output echo "$2" >expected - test_cmp output expected + test_cmp expected output } # @@ -35,34 +35,34 @@ test_kvs_key() { # test_expect_success 'kvs: integer put' ' - flux kvs put $KEY.integer=42 + flux kvs put --json $KEY.integer=42 ' test_expect_success 'kvs: double put' ' - flux kvs put $KEY.double=3.14 + flux kvs put --json $KEY.double=3.14 ' test_expect_success 'kvs: string put' ' - flux kvs put $KEY.string=foo + flux kvs put --json $KEY.string=foo ' test_expect_success 'kvs: empty string put' ' - flux kvs put $KEY.emptystring= + flux kvs put --json $KEY.emptystring= ' test_expect_success 'kvs: null is converted to json null' ' - flux kvs put $KEY.jsonnull=null + flux kvs put --json $KEY.jsonnull=null ' test_expect_success 'kvs: quoted null is converted to string' ' - flux kvs put $KEY.strnull=\"null\" + flux kvs put --json $KEY.strnull=\"null\" ' test_expect_success 'kvs: boolean true put' ' - flux kvs put $KEY.booleantrue=true + flux kvs put --json $KEY.booleantrue=true ' test_expect_success 'kvs: boolean false put' ' - flux kvs put $KEY.booleanfalse=false + flux kvs put --json $KEY.booleanfalse=false ' test_expect_success 'kvs: array put' ' - flux kvs put $KEY.array="[1,3,5]" + flux kvs put --json $KEY.array="[1,3,5]" ' test_expect_success 'kvs: object put' ' - flux kvs put $KEY.object="{\"a\":42}" + flux kvs put --json $KEY.object="{\"a\":42}" ' test_expect_success 'kvs: mkdir' ' flux kvs mkdir $SUBDIR1 @@ -139,43 +139,43 @@ EOF ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.integer && - test_must_fail flux kvs get $KEY.integer + test_must_fail flux kvs get --json $KEY.integer ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.double && - test_must_fail flux kvs get $KEY.double + test_must_fail flux kvs get --json $KEY.double ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.string && - test_must_fail flux kvs get $KEY.string + test_must_fail flux kvs get --json $KEY.string ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.emptystring && - test_must_fail flux kvs get $KEY.emptystring + test_must_fail flux kvs get --json $KEY.emptystring ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.jsonnull && - test_must_fail flux kvs get $KEY.jsonnull + test_must_fail flux kvs get --json $KEY.jsonnull ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.strnull && - test_must_fail flux kvs get $KEY.strnull + test_must_fail flux kvs get --json $KEY.strnull ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.booleantrue && - test_must_fail flux kvs get $KEY.booleantrue + test_must_fail flux kvs get --json $KEY.booleantrue ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.booleanfalse && - test_must_fail flux kvs get $KEY.booleanfalse + test_must_fail flux kvs get --json $KEY.booleanfalse ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.array && - test_must_fail flux kvs get $KEY.array + test_must_fail flux kvs get --json $KEY.array ' test_expect_success 'kvs: unlink works' ' flux kvs unlink $KEY.object && - test_must_fail flux kvs get $KEY.object + test_must_fail flux kvs get --json $KEY.object ' test_expect_success 'kvs: unlink dir works' ' flux kvs unlink $SUBDIR1 && @@ -191,10 +191,10 @@ test_expect_success 'kvs: unlink -R works' ' # test_expect_success 'kvs: put (multiple)' ' - flux kvs put $KEY.a=42 $KEY.b=3.14 $KEY.c=foo $KEY.d=true $KEY.e="[1,3,5]" $KEY.f="{\"a\":42}" + flux kvs put --json $KEY.a=42 $KEY.b=3.14 $KEY.c=foo $KEY.d=true $KEY.e="[1,3,5]" $KEY.f="{\"a\":42}" ' test_expect_success 'kvs: get (multiple)' ' - flux kvs get $KEY.a $KEY.b $KEY.c $KEY.d $KEY.e $KEY.f >output && + flux kvs get --json $KEY.a $KEY.b $KEY.c $KEY.d $KEY.e $KEY.f >output && cat >expected <output && cat >expected <output && cat >expected <output && @@ -296,7 +296,7 @@ test_expect_success 'kvs: ls -1F DIR works' ' ' test_expect_success 'kvs: ls -1Fd DIR.a DIR.b DIR.c works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a=69 && + flux kvs put --json $DIR.a=69 && flux kvs mkdir $DIR.b && flux kvs link b $DIR.c && flux kvs ls -1Fd $DIR.a $DIR.b $DIR.c >output && @@ -309,8 +309,8 @@ test_expect_success 'kvs: ls -1Fd DIR.a DIR.b DIR.c works' ' ' test_expect_success 'kvs: ls -1RF shows directory titles' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a=69 && - flux kvs put $DIR.b.d=42 && + flux kvs put --json $DIR.a=69 && + flux kvs put --json $DIR.b.d=42 && flux kvs link b $DIR.c && flux kvs ls -1RF $DIR | grep : | wc -l >output && cat >expected <<-EOF && @@ -387,7 +387,7 @@ test_expect_success 'kvs: ls key. works' ' ' test_expect_success 'kvs: ls key. fails if key is not a directory' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a=42 && + flux kvs put --json $DIR.a=42 && test_must_fail flux kvs ls -d $DIR.a. ' test_expect_success 'kvs: ls key. fails if key does not exist' ' @@ -395,16 +395,156 @@ test_expect_success 'kvs: ls key. fails if key does not exist' ' test_must_fail flux kvs ls $DIR.a ' +# +# zero-length values +# +test_expect_success 'kvs: zero-length value handled by put/get --raw' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a= && + flux kvs get --raw $DIR.a >empty.output && + test_cmp /dev/null empty.output +' +test_expect_success 'kvs: zero-length value handled by get with no options' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a= && + flux kvs get $DIR.a >empty2.output && + test_cmp /dev/null empty2.output +' +test_expect_success 'kvs: zero-length value handled by get --treeobj' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a= && + flux kvs get --treeobj $DIR.a +' +test_expect_success 'kvs: zero-length value NOT handled by get --json' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a= && + test_must_fail flux kvs get --json $DIR.a +' +test_expect_success 'kvs: zero-length value NOT made by put with no options' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a= && + flux kvs get --raw $DIR.a >onenull.output && + test_must_fail diff -q /dev/null onenull.output +' +test_expect_success 'kvs: zero-length value does not cause dir to fail' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a= && + flux kvs dir $DIR +' +test_expect_success 'kvs: zero-length value does not cause ls -FR to fail' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a= && + flux kvs ls -FR $DIR +' + +# +# empty string values +# +test_expect_success 'kvs: empty string value made by put with no options' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a= && + dd if=/dev/zero count=1 bs=1 of=empty.expected && + flux kvs get --raw $DIR.a >empty.actual && + test_cmp empty.expected empty.actual +' +test_expect_success 'kvs: empty string value can be retrieved by get' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a= && + echo >empty2.expected && + flux kvs get $DIR.a >empty2.actual && + test_cmp empty2.expected empty2.actual +' +test_expect_success 'kvs: empty string value NOT handled by get --json' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a= && + test_must_fail flux kvs get --json $DIR.a +' +test_expect_success 'kvs: empty string value does not cause dir to fail' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a= && + flux kvs dir $DIR | grep -q "a =" +' +test_expect_success 'kvs: empty string value does not cause ls -FR to fail' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a= && + flux kvs ls -FR $DIR | grep -q "a" +' + +# +# raw value tests +# +test_expect_success 'kvs: put/get --raw works with multiple key=val pairs' ' + flux kvs unlink -Rf $DIR && + flux kvs put --raw $DIR.a=xyz $DIR.b=zyx && + printf "%s" 'xyzzyx' >twovals.expected && + flux kvs get --raw $DIR.a $DIR.b >twovals.actual && + test_cmp twovals.expected twovals.actual +' +test_expect_success 'kvs: put --raw a=- reads value from stdin' ' + flux kvs unlink -Rf $DIR && + printf "%s" "abc" | flux kvs put --raw $DIR.a=- && + printf "%s" "abc" >rawstdin.expected && + flux kvs get --raw $DIR.a >rawstdin.actual && + test_cmp rawstdin.expected rawstdin.actual +' +test_expect_success 'kvs: put --raw a=- b=42 works' ' + flux kvs unlink -Rf $DIR && + printf "%s" "abc" | flux kvs put --raw $DIR.a=- $DIR.b=42 && + printf "%s" "abc42" >rawstdin2.expected && + flux kvs get --raw $DIR.a $DIR.b >rawstdin2.actual && + test_cmp rawstdin2.expected rawstdin2.actual +' +test_expect_success 'kvs: put --raw a=- b=- works with a getting all of stdin' ' + flux kvs unlink -Rf $DIR && + printf "%s" "abc" | flux kvs put --raw $DIR.a=- $DIR.b=- && + printf "%s" "abc" >rawstdin3a.expected && + flux kvs get --raw $DIR.a >rawstdin3a.actual && + flux kvs get --raw $DIR.b >rawstdin3b.actual && + test_cmp rawstdin3a.expected rawstdin3a.actual && + test_cmp /dev/null rawstdin3b.actual +' + +# +# treeobj tests +# +test_expect_success 'kvs: treeobj of all types handled by get --treeobj' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.val=hello && + flux kvs put $DIR.valref=$(seq -s 1 100) && + flux kvs mkdir $DIR.dirref && + flux kvs link foo $DIR.symlink && + flux kvs get --treeobj $DIR.val | grep -q val && + flux kvs get --treeobj $DIR.valref | grep -q valref && + flux kvs get --treeobj $DIR.dirref | grep -q dirref && + flux kvs get --treeobj $DIR.symlink | grep -q symlink +' +test_expect_success 'kvs: treeobj is created by put --treeobj' ' + flux kvs unlink -Rf $DIR && + flux kvs put --treeobj $DIR.val="{\"data\":\"YgA=\",\"type\":\"val\",\"ver\":1}" && + flux kvs get $DIR.val >val.output && + echo "b" >val.expected && + test_cmp val.expected val.output +' +test_expect_success 'kvs: treeobj can be used to create snapshot' ' + flux kvs unlink -Rf $DIR && + flux kvs put $DIR.a.a=hello && + flux kvs put $DIR.a.b=goodbye && + flux kvs put --treeobj $DIR.b=$(flux kvs get --treeobj $DIR.a) && + (flux kvs get $DIR.a.a && flux kvs get $DIR.a.b) >snap.expected && + (flux kvs get $DIR.b.a && flux kvs get $DIR.b.b) >snap.actual && + test_cmp snap.expected snap.actual +' + # # get corner case tests # test_expect_success 'kvs: get a nonexistent key' ' - test_must_fail flux kvs get NOT.A.KEY + test_must_fail flux kvs get --json NOT.A.KEY ' test_expect_success 'kvs: try to retrieve a directory as key should fail' ' flux kvs mkdir $DIR.a.b.c && - test_must_fail flux kvs get $DIR + test_must_fail flux kvs get --json $DIR ' # @@ -412,10 +552,10 @@ test_expect_success 'kvs: try to retrieve a directory as key should fail' ' # test_expect_success 'kvs: put with invalid input' ' - test_must_fail flux kvs put NOVALUE + test_must_fail flux kvs put --json NOVALUE ' test_expect_success 'kvs: put key of . fails' ' - test_must_fail flux kvs put .=1 + test_must_fail flux kvs put --json .=1 ' # @@ -428,7 +568,7 @@ test_empty_directory() { } test_expect_success 'kvs: try to retrieve key as directory should fail' ' - flux kvs put $DIR.a.b.c.d=42 && + flux kvs put --json $DIR.a.b.c.d=42 && test_must_fail flux kvs dir $DIR.a.b.c.d ' test_expect_success 'kvs: empty directory can be created' ' @@ -462,7 +602,7 @@ test_expect_success 'kvs: unlink -R works' ' ' test_expect_success 'kvs: empty directory remains after key removed' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a=1 && + flux kvs put --json $DIR.a=1 && test_kvs_key $DIR.a 1 && flux kvs unlink $DIR.a && test_empty_directory $DIR @@ -473,32 +613,32 @@ test_expect_success 'kvs: empty directory remains after key removed' ' # test_expect_success 'kvs: put with leading path separators works' ' flux kvs unlink -Rf $DIR && - flux kvs put ......$DIR.a.b.c=42 && + flux kvs put --json ......$DIR.a.b.c=42 && test_kvs_key $DIR.a.b.c 42 ' test_expect_success 'kvs: put with trailing path separators works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c........=43 && + flux kvs put --json $DIR.a.b.c........=43 && test_kvs_key $DIR.a.b.c 43 ' test_expect_success 'kvs: put with extra embedded path separators works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.....a....b...c=44 && + flux kvs put --json $DIR.....a....b...c=44 && test_kvs_key $DIR.a.b.c 44 ' test_expect_success 'kvs: get with leading path separators works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c=42 && + flux kvs put --json $DIR.a.b.c=42 && test_kvs_key ......$DIR.a.b.c 42 ' test_expect_success 'kvs: get with trailing path separators works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c=43 && + flux kvs put --json $DIR.a.b.c=43 && test_kvs_key $DIR.a.b.c........ 43 ' test_expect_success 'kvs: get with extra embedded path separators works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c=44 && + flux kvs put --json $DIR.a.b.c=44 && test_kvs_key $DIR.....a....b...c 44 ' @@ -508,14 +648,14 @@ test_expect_success 'kvs: get with extra embedded path separators works' ' test_expect_success 'kvs: link works' ' TARGET=$DIR.target && - flux kvs put $TARGET=\"foo\" && + flux kvs put --json $TARGET=\"foo\" && flux kvs link $TARGET $DIR.link && - OUTPUT=$(flux kvs get $DIR.link) && + OUTPUT=$(flux kvs get --json $DIR.link) && test "$OUTPUT" = "foo" ' test_expect_success 'kvs: readlink works' ' TARGET=$DIR.target && - flux kvs put $TARGET=\"foo\" && + flux kvs put --json $TARGET=\"foo\" && flux kvs link $TARGET $DIR.link && OUTPUT=$(flux kvs readlink $DIR.link) && test "$OUTPUT" = "$TARGET" @@ -523,8 +663,8 @@ test_expect_success 'kvs: readlink works' ' test_expect_success 'kvs: readlink works (multiple inputs)' ' TARGET1=$DIR.target1 && TARGET2=$DIR.target2 && - flux kvs put $TARGET1=\"foo1\" && - flux kvs put $TARGET2=\"foo2\" && + flux kvs put --json $TARGET1=\"foo1\" && + flux kvs put --json $TARGET2=\"foo2\" && flux kvs link $TARGET1 $DIR.link1 && flux kvs link $TARGET2 $DIR.link2 && flux kvs readlink $DIR.link1 $DIR.link2 >output && @@ -536,7 +676,7 @@ EOF ' test_expect_success 'kvs: readlink fails on regular value' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.target=42 && + flux kvs put --json $DIR.target=42 && ! flux kvs readlink $DIR.target ' test_expect_success 'kvs: readlink fails on directory' ' @@ -546,19 +686,19 @@ test_expect_success 'kvs: readlink fails on directory' ' ' test_expect_success 'kvs: link: path resolution when intermediate component is a link' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c=42 && + flux kvs put --json $DIR.a.b.c=42 && flux kvs link $DIR.a.b $DIR.Z.Y && - OUTPUT=$(flux kvs get $DIR.Z.Y.c) && + OUTPUT=$(flux kvs get --json $DIR.Z.Y.c) && test "$OUTPUT" = "42" ' test_expect_success 'kvs: link: path resolution with intermediate link and nonexistent key' ' flux kvs unlink -Rf $DIR && flux kvs link $DIR.a.b $DIR.Z.Y && - test_must_fail flux kvs get $DIR.Z.Y + test_must_fail flux kvs get --json $DIR.Z.Y ' test_expect_success 'kvs: link: intermediate link points to another link' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c=42 && + flux kvs put --json $DIR.a.b.c=42 && flux kvs link $DIR.a.b $DIR.Z.Y && flux kvs link $DIR.Z.Y $DIR.X.W && test_kvs_key $DIR.X.W.c 42 @@ -568,7 +708,7 @@ test_expect_success 'kvs: link: intermediate links are followed by put' ' flux kvs mkdir $DIR.a && flux kvs link $DIR.a $DIR.link && flux kvs readlink $DIR.link >/dev/null && - flux kvs put $DIR.link.X=42 && + flux kvs put --json $DIR.link.X=42 && flux kvs readlink $DIR.link >/dev/null && test_kvs_key $DIR.link.X 42 && test_kvs_key $DIR.a.X 42 @@ -579,7 +719,7 @@ test_expect_success 'kvs: link: kvs_copy removes linked destination' ' flux kvs unlink -Rf $DIR && flux kvs mkdir $DIR.a && flux kvs link $DIR.a $DIR.link && - flux kvs put $DIR.a.X=42 && + flux kvs put --json $DIR.a.X=42 && flux kvs copy $DIR.a $DIR.link && ! flux kvs readlink $DIR.link >/dev/null && test_kvs_key $DIR.link.X 42 @@ -590,7 +730,7 @@ test_expect_success 'kvs: link: kvs_move works' ' flux kvs unlink -Rf $DIR && flux kvs mkdir $DIR.a && flux kvs link $DIR.a $DIR.link && - flux kvs put $DIR.a.X=42 && + flux kvs put --json $DIR.a.X=42 && flux kvs move $DIR.a $DIR.link && ! flux kvs readlink $DIR.link >/dev/null && test_kvs_key $DIR.link.X 42 && @@ -599,7 +739,7 @@ test_expect_success 'kvs: link: kvs_move works' ' test_expect_success 'kvs: link: kvs_copy does not follow links (top)' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.X=42 && + flux kvs put --json $DIR.a.X=42 && flux kvs link $DIR.a $DIR.link && flux kvs copy $DIR.link $DIR.copy && LINKVAL=$(flux kvs readlink $DIR.copy) && @@ -608,7 +748,7 @@ test_expect_success 'kvs: link: kvs_copy does not follow links (top)' ' test_expect_success 'kvs: link: kvs_copy does not follow links (mid)' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.X=42 && + flux kvs put --json $DIR.a.b.X=42 && flux kvs link $DIR.a.b $DIR.a.link && flux kvs copy $DIR.a $DIR.copy && LINKVAL=$(flux kvs readlink $DIR.copy.link) && @@ -617,7 +757,7 @@ test_expect_success 'kvs: link: kvs_copy does not follow links (mid)' ' test_expect_success 'kvs: link: kvs_copy does not follow links (bottom)' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.X=42 && + flux kvs put --json $DIR.a.b.X=42 && flux kvs link $DIR.a.b.X $DIR.a.b.link && flux kvs copy $DIR.a $DIR.copy && LINKVAL=$(flux kvs readlink $DIR.copy.b.link) && @@ -635,7 +775,7 @@ test_expect_success 'kvs: link: readlink on dangling link' ' ' test_expect_success 'kvs: link: readlink works on non-dangling link' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.b.c="foo" && + flux kvs put --json $DIR.a.b.c="foo" && flux kvs link $DIR.a.b.c $DIR.link && OUTPUT=$(flux kvs readlink $DIR.link) && test "$OUTPUT" = "$DIR.a.b.c" @@ -645,7 +785,7 @@ test_expect_success 'kvs: link: readlink works on non-dangling link' ' test_expect_success 'kvs: link: error on link depth' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a=1 && + flux kvs put --json $DIR.a=1 && flux kvs link $DIR.a $DIR.b && flux kvs link $DIR.b $DIR.c && flux kvs link $DIR.c $DIR.d && @@ -657,14 +797,14 @@ test_expect_success 'kvs: link: error on link depth' ' flux kvs link $DIR.i $DIR.j && flux kvs link $DIR.j $DIR.k && flux kvs link $DIR.k $DIR.l && - test_must_fail flux kvs get $DIR.l + test_must_fail flux kvs get --json $DIR.l ' test_expect_success 'kvs: link: error on link depth, loop' ' flux kvs unlink -Rf $DIR && flux kvs link $DIR.link1 $DIR.link2 && flux kvs link $DIR.link2 $DIR.link1 && - test_must_fail flux kvs get $DIR.link1 + test_must_fail flux kvs get --json $DIR.link1 ' # @@ -672,20 +812,20 @@ test_expect_success 'kvs: link: error on link depth, loop' ' # test_expect_success 'kvs: copy works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.src=\"foo\" && + flux kvs put --json $DIR.src=\"foo\" && flux kvs copy $DIR.src $DIR.dest && - OUTPUT1=$(flux kvs get $DIR.src) && - OUTPUT2=$(flux kvs get $DIR.dest) && + OUTPUT1=$(flux kvs get --json $DIR.src) && + OUTPUT2=$(flux kvs get --json $DIR.dest) && test "$OUTPUT1" = "foo" && test "$OUTPUT2" = "foo" ' test_expect_success 'kvs: move works' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.src=\"foo\" && + flux kvs put --json $DIR.src=\"foo\" && flux kvs move $DIR.src $DIR.dest && - test_must_fail flux kvs get $DIR.src && - OUTPUT=$(flux kvs get $DIR.dest) && + test_must_fail flux kvs get --json $DIR.src && + OUTPUT=$(flux kvs get --json $DIR.dest) && test "$OUTPUT" = "foo" ' @@ -709,7 +849,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: version and wait' ' VERS=$((VERS + 1)) flux kvs wait $VERS & kvswaitpid=$! && - flux kvs put $DIR.xxx=99 && + flux kvs put --json $DIR.xxx=99 && test_expect_code 0 wait $kvswaitpid ' @@ -726,7 +866,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: version and wait' ' wait_watch_put() { i=0 - while [ "$(flux kvs get $1 2> /dev/null)" != "$2" ] && [ $i -lt 50 ] + while [ "$(flux kvs get --json $1 2> /dev/null)" != "$2" ] && [ $i -lt 50 ] do sleep 0.1 i=$((i + 1)) @@ -740,7 +880,7 @@ wait_watch_put() { wait_watch_empty() { i=0 - while flux kvs get $1 2> /dev/null && [ $i -lt 50 ] + while flux kvs get --json $1 2> /dev/null && [ $i -lt 50 ] do sleep 0.1 i=$((i + 1)) @@ -775,13 +915,13 @@ wait_watch_current() { test_expect_success NO_CHAIN_LINT 'kvs: watch a key' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.foo=0 && + flux kvs put --json $DIR.foo=0 && wait_watch_put "$DIR.foo" "0" rm -f watch_out stdbuf -oL flux kvs watch -o -c 1 $DIR.foo >watch_out & watchpid=$! && wait_watch_current "0" - flux kvs put $DIR.foo=1 && + flux kvs put --json $DIR.foo=1 && wait $watchpid cat >expected <<-EOF && 0 @@ -797,7 +937,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a key that at first doesnt exist' stdbuf -oL flux kvs watch -o -c 1 $DIR.foo >watch_out & watchpid=$! && wait_watch_current "nil" && - flux kvs put $DIR.foo=1 && + flux kvs put --json $DIR.foo=1 && wait $watchpid cat >expected <<-EOF && nil @@ -808,7 +948,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a key that at first doesnt exist' test_expect_success NO_CHAIN_LINT 'kvs: watch a key that gets removed' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.foo=0 && + flux kvs put --json $DIR.foo=0 && wait_watch_put "$DIR.foo" "0" rm -f watch_out stdbuf -oL flux kvs watch -o -c 1 $DIR.foo >watch_out & @@ -825,13 +965,13 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a key that gets removed' ' test_expect_success NO_CHAIN_LINT 'kvs: watch a key that becomes a dir' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.foo=0 && + flux kvs put --json $DIR.foo=0 && wait_watch_put "$DIR.foo" "0" rm -f watch_out stdbuf -oL flux kvs watch -o -c 1 $DIR.foo >watch_out & watchpid=$! && wait_watch_current "0" && - flux kvs put $DIR.foo.bar.baz=1 && + flux kvs put --json $DIR.foo.bar.baz=1 && wait $watchpid cat >expected <<-EOF && 0 @@ -844,14 +984,14 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a key that becomes a dir' ' test_expect_success NO_CHAIN_LINT 'kvs: watch a dir' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.a=0 $DIR.a.b=0 && + flux kvs put --json $DIR.a.a=0 $DIR.a.b=0 && wait_watch_put "$DIR.a.a" "0" && wait_watch_put "$DIR.a.b" "0" rm -f watch_out stdbuf -oL flux kvs watch -o -c 1 $DIR >watch_out & watchpid=$! && wait_watch_current "======================" && - flux kvs put $DIR.a.a=1 && + flux kvs put --json $DIR.a.a=1 && wait $watchpid cat >expected <<-EOF && $DIR.a. @@ -869,7 +1009,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a dir that at first doesnt exist' stdbuf -oL flux kvs watch -o -c 1 $DIR >watch_out & watchpid=$! && wait_watch_current "nil" && - flux kvs put $DIR.a.a=1 && + flux kvs put --json $DIR.a.a=1 && wait $watchpid cat >expected <<-EOF && nil @@ -882,7 +1022,7 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a dir that at first doesnt exist' test_expect_success NO_CHAIN_LINT 'kvs: watch a dir that gets removed' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.a.a=0 $DIR.a.a.b=0 && + flux kvs put --json $DIR.a.a.a=0 $DIR.a.a.b=0 && wait_watch_put "$DIR.a.a.a" "0" && wait_watch_put "$DIR.a.a.b" "0" rm -f watch_out @@ -902,14 +1042,14 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a dir that gets removed' ' test_expect_success NO_CHAIN_LINT 'kvs: watch a dir, converted into a key' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.a.a=0 $DIR.a.a.b=0 && + flux kvs put --json $DIR.a.a.a=0 $DIR.a.a.b=0 && wait_watch_put "$DIR.a.a.a" "0" && wait_watch_put "$DIR.a.a.b" "0" rm -f watch_out stdbuf -oL flux kvs watch -o -c 1 $DIR.a >watch_out & watchpid=$! && wait_watch_current "======================" && - flux kvs put $DIR.a=1 && + flux kvs put --json $DIR.a=1 && wait $watchpid cat >expected <<-EOF && $DIR.a.a. @@ -925,14 +1065,14 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a dir, converted into a key' ' # $DIR.a is no longer valid and we should see 'nil' as a result. test_expect_success NO_CHAIN_LINT 'kvs: watch a dir, prefix path converted into a key' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.a.a=0 $DIR.a.a.b=0 && + flux kvs put --json $DIR.a.a.a=0 $DIR.a.a.b=0 && wait_watch_put "$DIR.a.a.a" "0" && wait_watch_put "$DIR.a.a.b" "0" rm -f watch_out stdbuf -oL flux kvs watch -o -c 1 $DIR.a >watch_out & watchpid=$! && wait_watch_current "======================" && - flux kvs put $DIR=1 && + flux kvs put --json $DIR=1 && wait $watchpid cat >expected <<-EOF && $DIR.a.a. @@ -965,14 +1105,14 @@ sort_watch_output() { test_expect_success NO_CHAIN_LINT 'kvs: watch a dir with -R' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.a=0 $DIR.a.b=0 && + flux kvs put --json $DIR.a.a=0 $DIR.a.b=0 && wait_watch_put "$DIR.a.a" "0" && wait_watch_put "$DIR.a.b" "0" rm -f watch_out stdbuf -oL flux kvs watch -R -o -c 1 $DIR >watch_out & watchpid=$! && wait_watch_current "======================" && - flux kvs put $DIR.a.a=1 && + flux kvs put --json $DIR.a.a=1 && wait $watchpid sort_watch_output cat >expected <<-EOF && @@ -988,14 +1128,14 @@ test_expect_success NO_CHAIN_LINT 'kvs: watch a dir with -R' ' test_expect_success NO_CHAIN_LINT 'kvs: watch a dir with -R and -d' ' flux kvs unlink -Rf $DIR && - flux kvs put $DIR.a.a=0 $DIR.a.b=0 && + flux kvs put --json $DIR.a.a=0 $DIR.a.b=0 && wait_watch_put "$DIR.a.a" "0" && wait_watch_put "$DIR.a.b" "0" rm -f watch_out stdbuf -oL flux kvs watch -R -d -o -c 1 $DIR >watch_out & watchpid=$! && wait_watch_current "======================" && - flux kvs put $DIR.a.a=1 && + flux kvs put --json $DIR.a.a=1 && wait $watchpid sort_watch_output cat >expected <<-EOF && diff --git a/t/t1002-kvs-extra.t b/t/t1002-kvs-extra.t index 71d8b931c140..57dd8e2b0a02 100755 --- a/t/t1002-kvs-extra.t +++ b/t/t1002-kvs-extra.t @@ -29,7 +29,7 @@ KEY=test.a.b.c # # test_kvs_key() { - flux kvs get "$1" >output + flux kvs get --json "$1" >output echo "$2" >expected test_cmp output expected #if ! test "$OUTPUT" = "$2"; then @@ -45,24 +45,24 @@ test_kvs_type () { } test_expect_success 'kvs: integer put' ' - flux kvs put $KEY=42 + flux kvs put --json $KEY=42 ' test_expect_success 'kvs: integer type' ' test_kvs_type $KEY int ' test_expect_success 'kvs: value can be empty' ' - flux kvs put $KEY= && + flux kvs put --json $KEY= && test_kvs_key $KEY "" && test_kvs_type $KEY string ' test_expect_success 'kvs: null is converted to json null' ' - flux kvs put $KEY=null && + flux kvs put --json $KEY=null && test_kvs_key $KEY nil && test_kvs_type $KEY null ' test_expect_success 'kvs: quoted null is converted to string' ' - flux kvs put $KEY=\"null\" && + flux kvs put --json $KEY=\"null\" && test_kvs_key $KEY null && test_kvs_type $KEY string ' @@ -70,19 +70,19 @@ test_expect_success 'kvs: quoted null is converted to string' ' KEY=$TEST.b.c.d DIR=$TEST.b.c test_expect_success 'kvs: string put' ' - flux kvs put $KEY="Hello world" + flux kvs put --json $KEY="Hello world" ' test_expect_success 'kvs: string type' ' test_kvs_type $KEY string ' test_expect_success 'kvs: boolean put' ' - flux kvs put $KEY=true + flux kvs put --json $KEY=true ' test_expect_success 'kvs: boolean type' ' test_kvs_type $KEY boolean ' test_expect_success 'kvs: put double' ' - flux kvs put $KEY=3.14159 + flux kvs put --json $KEY=3.14159 ' test_expect_success 'kvs: double type' ' test_kvs_type $KEY double @@ -90,7 +90,7 @@ test_expect_success 'kvs: double type' ' # issue 875 test_expect_success 'kvs: integer can be read as int, int64, or double' ' - flux kvs put $TEST.a=2 && + flux kvs put --json $TEST.a=2 && test_kvs_type $TEST.a int && test $($GETAS -t int $TEST.a) = "2" && test $($GETAS -t int -d $TEST a) = "2" && @@ -100,25 +100,25 @@ test_expect_success 'kvs: integer can be read as int, int64, or double' ' test $($GETAS -t double -d $TEST a | cut -d. -f1) = "2" ' test_expect_success 'kvs: array put' ' - flux kvs put $KEY="[1,3,5,7]" + flux kvs put --json $KEY="[1,3,5,7]" ' test_expect_success 'kvs: array type' ' test_kvs_type $KEY array ' test_expect_success 'kvs: object put' ' - flux kvs put $KEY="{\"a\":42}" + flux kvs put --json $KEY="{\"a\":42}" ' test_expect_success 'kvs: object type' ' test_kvs_type $KEY object ' -test_expect_success 'kvs: put using no-merge flag' ' +test_expect_success 'kvs: put using --no-merge flag' ' flux kvs unlink -Rf $TEST && - ${KVSBASIC} put-no-merge $DIR.a=69 && - ${KVSBASIC} put-no-merge $DIR.b.c.d.e.f.g=70 && - ${KVSBASIC} put-no-merge $DIR.c.a.b=3.14 && - ${KVSBASIC} put-no-merge $DIR.d=\"snerg\" && - ${KVSBASIC} put-no-merge $DIR.e=true && + flux kvs put --no-merge --json $DIR.a=69 && + flux kvs put --no-merge --json $DIR.b.c.d.e.f.g=70 && + flux kvs put --no-merge --json $DIR.c.a.b=3.14 && + flux kvs put --no-merge --json $DIR.d=\"snerg\" && + flux kvs put --no-merge --json $DIR.e=true && flux kvs dir -R $DIR | sort >output && cat >expected <output && + flux kvs put --json $DIR.a=69 && + flux kvs put --json $DIR.b.c.d.e.f.g=70 && + flux kvs put --json $DIR.c.a.b=3.14 && + flux kvs put --json $DIR.d=\"snerg\" && + flux kvs put --json $DIR.e=true && + DIRREF=$(flux kvs get --treeobj $DIR) && + flux kvs dir -R --at $DIRREF . | sort >output && cat >expected <snapshot && - ${KVSBASIC} put-treeobj $TEST.snap="`cat snapshot`" && - ${KVSBASIC} get-treeobj $TEST.snap >snapshot.cpy && + flux kvs get --treeobj . >snapshot && + flux kvs put --treeobj $TEST.snap="`cat snapshot`" && + flux kvs get --treeobj $TEST.snap >snapshot.cpy && test_cmp snapshot snapshot.cpy ' -test_expect_success 'kvs: put-treeobj: clobbers destination' ' +test_expect_success 'kvs: put --treeobj: clobbers destination' ' flux kvs unlink -Rf $TEST && - flux kvs put $TEST.a=42 && - ${KVSBASIC} get-treeobj . >snapshot2 && - ${KVSBASIC} put-treeobj $TEST.a="`cat snapshot2`" && - ! flux kvs get $TEST.a && + flux kvs put --json $TEST.a=42 && + flux kvs get --treeobj . >snapshot2 && + flux kvs put --treeobj $TEST.a="`cat snapshot2`" && + ! flux kvs get --json $TEST.a && flux kvs dir $TEST.a ' -test_expect_success 'kvs: put-treeobj: fails bad dirent: not JSON' ' +test_expect_success 'kvs: put --treeobj: fails bad dirent: not JSON' ' flux kvs unlink -Rf $TEST && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a=xyz + test_must_fail flux kvs put --treeobj $TEST.a=xyz ' -test_expect_success 'kvs: put-treeobj: fails bad dirent: unknown type' ' +test_expect_success 'kvs: put --treeobj: fails bad dirent: unknown type' ' flux kvs unlink -Rf $TEST && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a="{\"data\":\"MQA=\",\"type\":\"FOO\",\"ver\":1}" + test_must_fail flux kvs put --treeobj $TEST.a="{\"data\":\"MQA=\",\"type\":\"FOO\",\"ver\":1}" ' -test_expect_success 'kvs: put-treeobj: fails bad dirent: bad link data' ' +test_expect_success 'kvs: put --treeobj: fails bad dirent: bad link data' ' flux kvs unlink -Rf $TEST && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a="{\"data\":42,\"type\":\"symlink\",\"ver\":1}" + test_must_fail flux kvs put --treeobj $TEST.a="{\"data\":42,\"type\":\"symlink\",\"ver\":1}" ' -test_expect_success 'kvs: put-treeobj: fails bad dirent: bad ref data' ' +test_expect_success 'kvs: put --treeobj: fails bad dirent: bad ref data' ' flux kvs unlink -Rf $TEST && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a="{\"data\":42,\"type\":\"dirref\",\"ver\":1}" && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a="{\"data\":"sha1-4087718d190b373fb490b27873f61552d7f29dbe",\"type\":\"dirref\",\"ver\":1}" + test_must_fail flux kvs put --treeobj $TEST.a="{\"data\":42,\"type\":\"dirref\",\"ver\":1}" && + test_must_fail flux kvs put --treeobj $TEST.a="{\"data\":"sha1-4087718d190b373fb490b27873f61552d7f29dbe",\"type\":\"dirref\",\"ver\":1}" ' -test_expect_success 'kvs: put-treeobj: fails bad dirent: bad blobref' ' +test_expect_success 'kvs: put --treeobj: fails bad dirent: bad blobref' ' flux kvs unlink -Rf $TEST && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a="{\"data\":[\"sha1-aaa\"],\"type\":\"dirref\",\"ver\":1}" && - test_must_fail ${KVSBASIC} put-treeobj $TEST.a="{\"data\":[\"sha1-bbb\"],\"type\":\"dirref\",\"ver\":1}" + test_must_fail flux kvs put --treeobj $TEST.a="{\"data\":[\"sha1-aaa\"],\"type\":\"dirref\",\"ver\":1}" && + test_must_fail flux kvs put --treeobj $TEST.a="{\"data\":[\"sha1-bbb\"],\"type\":\"dirref\",\"ver\":1}" ' -test_expect_success 'kvs: getat: fails bad on dirent' ' +test_expect_success 'kvs: get --at: fails bad on dirent' ' flux kvs unlink -Rf $TEST && - test_must_fail ${KVSBASIC} getat 42 $TEST.a && - test_must_fail ${KVSBASIC} getat "{\"data\":[\"sha1-aaa\"],\"type\":\"dirref\",\"ver\":1}" $TEST.a && - test_must_fail ${KVSBASIC} getat "{\"data\":[\"sha1-bbb\"],\"type\":\"dirref\",\"ver\":1}" $TEST.a && - test_must_fail ${KVSBASIC} getat "{\"data\":42,\"type\":\"dirref\",\"ver\":1}" $TEST.a && - test_must_fail ${KVSBASIC} getat "{\"data\":"sha1-4087718d190b373fb490b27873f61552d7f29dbe",\"type\":\"dirref\",\"ver\":1}" $TEST.a + test_must_fail flux kvs get --at 42 $TEST.a && + test_must_fail flux kvs get --at "{\"data\":[\"sha1-aaa\"],\"type\":\"dirref\",\"ver\":1}" $TEST.a && + test_must_fail flux kvs get --at "{\"data\":[\"sha1-bbb\"],\"type\":\"dirref\",\"ver\":1}" $TEST.a && + test_must_fail flux kvs get --at "{\"data\":42,\"type\":\"dirref\",\"ver\":1}" $TEST.a && + test_must_fail flux kvs get --at "{\"data\":"sha1-4087718d190b373fb490b27873f61552d7f29dbe",\"type\":\"dirref\",\"ver\":1}" $TEST.a ' -test_expect_success 'kvs: getat: works on root from get-treeobj' ' +test_expect_success 'kvs: get --at: works on root from get --treeobj' ' flux kvs unlink -Rf $TEST && - flux kvs put $TEST.a.b.c=42 && - test $(${KVSBASIC} getat $(${KVSBASIC} get-treeobj .) $TEST.a.b.c) = 42 + flux kvs put --json $TEST.a.b.c=42 && + test $(flux kvs get --at $(flux kvs get --treeobj .) $TEST.a.b.c) = 42 ' -test_expect_success 'kvs: getat: works on subdir from get-treeobj' ' +test_expect_success 'kvs: get --at: works on subdir from get --treeobj' ' flux kvs unlink -Rf $TEST && - flux kvs put $TEST.a.b.c=42 && - test $(${KVSBASIC} getat $(${KVSBASIC} get-treeobj $TEST.a.b) c) = 42 + flux kvs put --json $TEST.a.b.c=42 && + test $(flux kvs get --at $(flux kvs get --treeobj $TEST.a.b) c) = 42 ' -test_expect_success 'kvs: getat: works on outdated root' ' +test_expect_success 'kvs: get --at: works on outdated root' ' flux kvs unlink -Rf $TEST && - flux kvs put $TEST.a.b.c=42 && - ROOTREF=$(${KVSBASIC} get-treeobj .) && - flux kvs put $TEST.a.b.c=43 && - test $(${KVSBASIC} getat $ROOTREF $TEST.a.b.c) = 42 -' - -test_expect_success 'kvs: zero size raw value can be stored and retrieved' ' - flux kvs unlink -Rf $TEST && - ${KVSBASIC} copy-tokvs $TEST.empty - random.data && - ${KVSBASIC} copy-tokvs $TEST.data random.data && - ${KVSBASIC} copy-fromkvs $TEST.data reread.data && + flux kvs put --raw $TEST.data=- reread.data && test_cmp random.data reread.data ' diff --git a/t/t2000-wreck.t b/t/t2000-wreck.t index 95644ba90806..6ac1c1cb740a 100755 --- a/t/t2000-wreck.t +++ b/t/t2000-wreck.t @@ -29,8 +29,8 @@ test_expect_success 'wreckrun: -T, --walltime works' ' test_expect_code 142 flux wreckrun --walltime=1s -n${SIZE} sleep 15 ' test_expect_success 'wreckrun: -T, --walltime allows override of default signal' ' - flux kvs put lwj.walltime-signal=SIGTERM && - test_when_finished flux kvs put lwj.walltime-signal= && + flux kvs put --json lwj.walltime-signal=SIGTERM && + test_when_finished flux kvs put --json lwj.walltime-signal= && test_expect_code 143 flux wreckrun -T 1s -n${SIZE} sleep 5 ' test_expect_success 'wreckrun: -T, --walltime allows per-job override of default signal' ' @@ -182,25 +182,25 @@ test_expect_success 'wreckrun: -N without -n works' ' test_expect_success 'wreckrun: -N without -n sets ntasks in kvs' ' flux wreckrun -l -N${SIZE} /bin/true && LWJ=$(last_job_path) && - n=$(flux kvs get ${LWJ}.ntasks) && + n=$(flux kvs get --json ${LWJ}.ntasks) && test "$n" = "${SIZE}" ' test_expect_success 'wreckrun: -n without -N sets nnnodes in kvs' ' flux wreckrun -l -n${SIZE} /bin/true && LWJ=$(last_job_path) && - n=$(flux kvs get ${LWJ}.nnodes) && + n=$(flux kvs get --json ${LWJ}.nnodes) && test "$n" = "${SIZE}" ' test_expect_success 'wreckrun: -t1 -N${SIZE} sets ntasks in kvs' ' flux wreckrun -l -t1 -N${SIZE} /bin/true && LWJ=$(last_job_path) && - n=$(flux kvs get ${LWJ}.ntasks) && + n=$(flux kvs get --json ${LWJ}.ntasks) && test "$n" = "${SIZE}" ' test_expect_success 'wreckrun: -t1 -n${SIZE} sets nnodes in kvs' ' flux wreckrun -l -t1 -n${SIZE} /bin/true && LWJ=$(last_job_path) && - n=$(flux kvs get ${LWJ}.nnodes) && + n=$(flux kvs get --json ${LWJ}.nnodes) && test "$n" = "${SIZE}" ' @@ -230,7 +230,7 @@ test_expect_success MULTICORE 'wreckrun: supports per-task affinity assignment' test_cmp expected_cpus2 output_cpus2 ' test_expect_success 'wreckrun: top level environment' ' - flux kvs put lwj.environ="{ \"TEST_ENV_VAR\": \"foo\" }" && + flux kvs put --json lwj.environ="{ \"TEST_ENV_VAR\": \"foo\" }" && run_timeout 5 flux wreckrun -n2 printenv TEST_ENV_VAR > output_top_env && cat <<-EOF >expected_top_env && foo @@ -285,9 +285,9 @@ test_expect_success 'wreckrun: --error supported' ' $WAITFILE -v --timeout=1 -p "this is stdout" test2.out ' test_expect_success 'wreckrun: kvs config output for all jobs' ' - test_when_finished "flux kvs put lwj.output=" && - flux kvs put lwj.output.labelio=true && - flux kvs put lwj.output.files.stdout=test3.out && + test_when_finished "flux kvs put --json lwj.output=" && + flux kvs put --json lwj.output.labelio=true && + flux kvs put --json lwj.output.files.stdout=test3.out && flux wreckrun -n${SIZE} echo foo && for i in $(seq 0 $((${SIZE}-1))) do echo "$i: foo" @@ -374,7 +374,7 @@ test_expect_success NO_SCHED 'flux-submit: returns ENOSYS when sched not loaded' check_complete_link() { for i in `seq 0 5`; do lastepoch=$(flux kvs dir lwj-complete | awk -F. '{print $2}' | sort -n | tail -1) - flux kvs get lwj-complete.${lastepoch}.${1}.state && return 0 + flux kvs get --json lwj-complete.${lastepoch}.${1}.state && return 0 sleep 0.2 done return 1 diff --git a/t/t2001-jsc.t b/t/t2001-jsc.t index e5585b1e5adb..5901e732337e 100755 --- a/t/t2001-jsc.t +++ b/t/t2001-jsc.t @@ -208,7 +208,7 @@ test_expect_success 'jstat 8: query detects bad inputs' ' test_expect_success 'jstat 9: update state-pair' " flux jstat update 1 state-pair '{\"state-pair\": {\"ostate\": 13, \"nstate\": 12}}' && - flux kvs get $(flux wreck kvs-path 1).state > output.9.1 && + flux kvs get --json $(flux wreck kvs-path 1).state > output.9.1 && cat >expected.9.1 <<-EOF && cancelled EOF @@ -216,15 +216,15 @@ EOF " test_expect_success 'jstat 10: update procdescs' " - flux kvs get $(flux wreck kvs-path 1).0.procdesc > output.10.1 && + flux kvs get --json $(flux wreck kvs-path 1).0.procdesc > output.10.1 && flux jstat update 1 pdesc '{\"pdesc\": {\"procsize\":1, \"hostnames\":[\"0\"], \"executables\":[\"fake\"], \"pdarray\":[{\"pid\":8482,\"eindx\":0,\"hindx\":0}]}}' && - flux kvs get $(flux wreck kvs-path 1).0.procdesc > output.10.2 && + flux kvs get --json $(flux wreck kvs-path 1).0.procdesc > output.10.2 && test_expect_code 1 diff output.10.1 output.10.2 " test_expect_success 'jstat 11: update rdesc' " flux jstat update 1 rdesc '{\"rdesc\": {\"nnodes\": 128, \"ntasks\": 128, \"walltime\":3600}}' && - flux kvs get $(flux wreck kvs-path 1).ntasks > output.11.1 && + flux kvs get --json $(flux wreck kvs-path 1).ntasks > output.11.1 && cat > expected.11.1 <<-EOF && 128 EOF @@ -233,7 +233,7 @@ EOF test_expect_success 'jstat 12: update rdl' " flux jstat update 1 rdl '{\"rdl\": \"fake_rdl_string\"}' && - flux kvs get $(flux wreck kvs-path 1).rdl > output.12.1 && + flux kvs get --json $(flux wreck kvs-path 1).rdl > output.12.1 && cat > expected.12.1 <<-EOF && fake_rdl_string EOF @@ -242,7 +242,7 @@ EOF test_expect_success 'jstat 13: update rdl_alloc' " flux jstat update 1 rdl_alloc '{\"rdl_alloc\": [{\"contained\": {\"cmbdrank\": 0, \"cmbdncores\": 102}}]}' && - flux kvs get $(flux wreck kvs-path 1).rank.0.cores > output.13.1 && + flux kvs get --json $(flux wreck kvs-path 1).rank.0.cores > output.13.1 && cat > expected.13.1 <<-EOF && 102 EOF diff --git a/t/t2002-pmi.t b/t/t2002-pmi.t index 6c25e58a8a35..c945e7ce423f 100755 --- a/t/t2002-pmi.t +++ b/t/t2002-pmi.t @@ -93,7 +93,7 @@ test_expect_success 'pmi: wreck preputs PMI_process_mapping into kvs' ' #!/bin/sh if test \${FLUX_TASK_RANK} -eq 0; then KVS_PATH=\$(flux wreck kvs-path \${FLUX_JOB_ID}) - flux kvs get \${KVS_PATH}.pmi.PMI_process_mapping + flux kvs get --json \${KVS_PATH}.pmi.PMI_process_mapping fi EOF chmod +x print-pmi-map.sh && diff --git a/t/t2005-hwloc-basic.t b/t/t2005-hwloc-basic.t index d8c4ea04cea4..cc79aa606fe5 100755 --- a/t/t2005-hwloc-basic.t +++ b/t/t2005-hwloc-basic.t @@ -115,16 +115,16 @@ test_expect_success HAVE_LSTOPO 'hwloc: test failure of lstopo command' ' ' test_expect_success 'hwloc: no broken down resource info by default' ' - test_must_fail flux kvs get resource.hwloc.by_rank.0.Machine_0.OSName + test_must_fail flux kvs get --json resource.hwloc.by_rank.0.Machine_0.OSName ' test_expect_success 'hwloc: reload --walk-topology=yes works' ' flux hwloc reload --walk-topology=yes && - flux kvs get resource.hwloc.by_rank.0.Machine_0.OSName + flux kvs get --json resource.hwloc.by_rank.0.Machine_0.OSName ' test_expect_success 'hwloc: reload --walk-topology=no removes broken down topo' ' flux hwloc reload --walk-topology=no && - test_must_fail flux kvs get resource.hwloc.by_rank.0.Machine_0.OSName + test_must_fail flux kvs get --json resource.hwloc.by_rank.0.Machine_0.OSName ' test_expect_success 'hwloc: reload fails on invalid rank' ' @@ -133,7 +133,7 @@ test_expect_success 'hwloc: reload fails on invalid rank' ' ' test_expect_success 'hwloc: HostName is populated in by_rank' ' - HW_HOST=$(flux kvs get resource.hwloc.by_rank.0.HostName) && + HW_HOST=$(flux kvs get --json resource.hwloc.by_rank.0.HostName) && REAL_HOST=$(hostname) && test x"$HW_HOST" = x"$REAL_HOST" '