diff --git a/src/bindings/python/flux/job.py b/src/bindings/python/flux/job.py index 88e4723bffe1..5a31bc0ef3ba 100644 --- a/src/bindings/python/flux/job.py +++ b/src/bindings/python/flux/job.py @@ -86,6 +86,88 @@ def job_kvs_guest(flux_handle, jobid): return flux.kvs.get_dir(flux_handle, kvs_key) +def id_parse(jobid_str): + """ + returns: An integer jobid + :rtype int + """ + jobid = ffi.new("flux_jobid_t[1]") + RAW.id_parse(jobid_str, jobid) + return int(jobid[0]) + + +def id_encode(jobid, encoding="f58"): + """ + returns: Jobid encoded in encoding + :rtype str + """ + buflen = 128 + buf = ffi.new("char[]", buflen) + RAW.id_encode(int(jobid), encoding, buf, buflen) + return ffi.string(buf, buflen).decode("utf-8") + + +class JobID(int): + """Class used to represent a Flux JOBID + + JobID is a subclass of `int`, so may be used in place of integer. + However, a JobID may be created from any valid RFC 19 FLUID + encoding, including: + + - decimal integer (no prefix) + - hexidecimal integer (prefix 0x) + - dotted hex (dothex) (xxxx.xxxx.xxxx.xxxx) + - kvs dir (dotted hex with `job.` prefix) + - RFC19 F58: (Base58 encoding with prefix `ƒ` or `f`) + + A JobID object also has properties for encoding a JOBID into each + of the above representations, e.g. jobid.f85, jobid.words, jobid.dothex... + + """ + + def __new__(cls, value, *args, **kwargs): + if isinstance(value, int): + jobid = value + else: + jobid = id_parse(value) + return super(cls, cls).__new__(cls, jobid) + + def encode(self, encoding="dec"): + """Encode a JobID to alternate supported format""" + return id_encode(self, encoding) + + @property + def f58(self): + """Return RFC19 F58 representation of a JobID""" + return self.encode("f58") + + @property + def hex(self): + """Return 0x-prefixed hexidecimal representation of a JobID""" + return self.encode("hex") + + @property + def dothex(self): + """Return dotted hexidecimal representation of a JobID""" + return self.encode("dothex") + + @property + def words(self): + """Return words (mnemonic) representation of a JobID""" + return self.encode("words") + + @property + def kvs(self): + """Return KVS directory path of a JobID""" + return self.encode("kvs") + + def __str__(self): + return self.encode() + + def __repr__(self): + return f"JobID({self})" + + class SubmitFuture(Future): def get_id(self): return submit_get_id(self) diff --git a/src/bindings/python/flux/wrapper.py b/src/bindings/python/flux/wrapper.py index 13632f9142d3..fa444164d24a 100644 --- a/src/bindings/python/flux/wrapper.py +++ b/src/bindings/python/flux/wrapper.py @@ -190,7 +190,7 @@ def __call__(self, calling_object, *args_in): # Unpack wrapper objects args[i] = args[i].handle elif isinstance(args[i], six.text_type): - args[i] = args[i].encode("utf-8") + args[i] = args[i].encode("utf-8", errors="surrogateescape") try: result = self.fun(*args) diff --git a/src/cmd/flux-job.c b/src/cmd/flux-job.c index 9f4956842482..a1c159cf1e71 100644 --- a/src/cmd/flux-job.c +++ b/src/cmd/flux-job.c @@ -33,7 +33,6 @@ #endif #include "src/common/libutil/xzmalloc.h" #include "src/common/libutil/log.h" -#include "src/common/libutil/fluid.h" #include "src/common/libjob/job.h" #include "src/common/libutil/read_all.h" #include "src/common/libutil/monotime.h" @@ -244,12 +243,8 @@ static struct optparse_option status_opts[] = { }; static struct optparse_option id_opts[] = { - { .name = "from", .key = 'f', .has_arg = 1, - .arginfo = "dec|kvs|hex|words", - .usage = "Convert jobid from specified form", - }, { .name = "to", .key = 't', .has_arg = 1, - .arginfo = "dec|kvs|hex|words", + .arginfo = "dec|kvs|hex|dothex|words|f58", .usage = "Convert jobid to specified form", }, OPTPARSE_TABLE_END @@ -510,6 +505,14 @@ int main (int argc, char *argv[]) return (exitval); } +static flux_jobid_t parse_jobid (const char *s) +{ + flux_jobid_t id; + if (flux_job_id_parse (s, &id) < 0) + log_msg_exit ("error parsing jobid: \"%s\"", s); + return id; +} + /* Parse a free argument 's', expected to be a 64-bit unsigned. * On error, exit complaining about parsing 'name'. */ @@ -609,7 +612,7 @@ int cmd_priority (optparse_t *p, int argc, char **argv) if (!(h = flux_open (NULL, 0))) log_err_exit ("flux_open"); - id = parse_arg_unsigned (argv[optindex++], "jobid"); + id = parse_jobid (argv[optindex++]); priority = parse_arg_unsigned (argv[optindex++], "priority"); if (!(f = flux_job_set_priority (h, id, priority))) @@ -636,7 +639,7 @@ int cmd_raise (optparse_t *p, int argc, char **argv) exit (1); } - id = parse_arg_unsigned (argv[optindex++], "jobid"); + id = parse_jobid (argv[optindex++]); if (optindex < argc) note = parse_arg_message (argv + optindex, "message"); @@ -842,7 +845,7 @@ int cmd_kill (optparse_t *p, int argc, char **argv) exit (1); } - id = parse_arg_unsigned (argv[optindex++], "jobid"); + id = parse_jobid (argv[optindex++]); s = optparse_get_str (p, "signal", "SIGTERM"); if ((signum = str2signum (s))< 0) @@ -932,7 +935,7 @@ int cmd_cancel (optparse_t *p, int argc, char **argv) exit (1); } - id = parse_arg_unsigned (argv[optindex++], "jobid"); + id = parse_jobid (argv[optindex++]); if (optindex < argc) note = parse_arg_message (argv + optindex, "message"); @@ -1118,7 +1121,7 @@ int cmd_list_ids (optparse_t *p, int argc, char **argv) ids_len = argc - optindex; for (i = 0; i < ids_len; i++) { - flux_jobid_t id = parse_arg_unsigned (argv[optindex + i], "id"); + flux_jobid_t id = parse_jobid (argv[optindex + i]); flux_future_t *f; if (!(f = flux_job_list_id (h, id, list_attrs))) log_err_exit ("flux_job_list_id"); @@ -2020,7 +2023,7 @@ int cmd_attach (optparse_t *p, int argc, char **argv) optparse_print_usage (p); exit (1); } - ctx.id = parse_arg_unsigned (argv[optindex++], "jobid"); + ctx.id = parse_jobid (argv[optindex++]); ctx.p = p; if (!(ctx.h = flux_open (NULL, 0))) @@ -2191,7 +2194,7 @@ int cmd_status (optparse_t *p, int argc, char **argv) for (i = 0; i < njobs; i++) { struct job_status *stat = &stats[i]; - stat->id = parse_arg_unsigned (argv[optindex+i], "jobid"); + stat->id = parse_jobid (argv[optindex+i]); stat->exception_exit_code = exception_exit_code; if (!(f = flux_job_event_watch (h, stat->id, "eventlog", 0))) @@ -2239,51 +2242,21 @@ int cmd_status (optparse_t *p, int argc, char **argv) void id_convert (optparse_t *p, const char *src, char *dst, int dstsz) { - const char *from = optparse_get_str (p, "from", "dec"); const char *to = optparse_get_str (p, "to", "dec"); flux_jobid_t id; - /* src to id + /* Parse as any valid JOBID */ - if (!strcmp (from, "dec")) { - id = parse_arg_unsigned (src, "input"); - } - else if (!strcmp (from, "hex")) { - if (fluid_decode (src, &id, FLUID_STRING_DOTHEX) < 0) - log_msg_exit ("%s: malformed input", src); - } - else if (!strcmp (from, "kvs")) { - if (strncmp (src, "job.", 4) != 0) - log_msg_exit ("%s: missing 'job.' prefix", src); - if (fluid_decode (src + 4, &id, FLUID_STRING_DOTHEX) < 0) - log_msg_exit ("%s: malformed input", src); - } - else if (!strcmp (from, "words")) { - if (fluid_decode (src, &id, FLUID_STRING_MNEMONIC) < 0) - log_msg_exit ("%s: malformed input", src); - } - else - log_msg_exit ("Unknown from=%s", from); + if (flux_job_id_parse (src, &id) < 0) + log_msg_exit ("%s: malformed input", src); - /* id to dst + /* Now encode into requested representation: */ - if (!strcmp (to, "dec")) { - snprintf (dst, dstsz, "%ju", (uintmax_t) id); - } - else if (!strcmp (to, "kvs")) { - if (flux_job_kvs_key (dst, dstsz, id, NULL) < 0) - log_msg_exit ("error encoding id"); + if (flux_job_id_encode (id, to, dst, dstsz) < 0) { + if (errno == EPROTO) + log_msg_exit ("Unknown to=%s", to); + log_msg_exit ("Unable to encode id %ju to %s", (uintmax_t) id, to); } - else if (!strcmp (to, "hex")) { - if (fluid_encode (dst, dstsz, id, FLUID_STRING_DOTHEX) < 0) - log_msg_exit ("error encoding id"); - } - else if (!strcmp (to, "words")) { - if (fluid_encode (dst, dstsz, id, FLUID_STRING_MNEMONIC) < 0) - log_msg_exit ("error encoding id"); - } - else - log_msg_exit ("Unknown to=%s", to); } char *trim_string (char *s) @@ -2325,7 +2298,7 @@ int cmd_id (optparse_t *p, int argc, char **argv) static void print_job_namespace (const char *src) { char ns[64]; - flux_jobid_t id = parse_arg_unsigned (src, "jobid"); + flux_jobid_t id = parse_jobid (src); if (flux_job_kvs_namespace (ns, sizeof (ns), id) < 0) log_msg_exit ("error getting kvs namespace for %ju", id); printf ("%s\n", ns); @@ -2501,7 +2474,7 @@ int cmd_eventlog (optparse_t *p, int argc, char **argv) exit (1); } - ctx.id = parse_arg_unsigned (argv[optindex++], "jobid"); + ctx.id = parse_jobid (argv[optindex++]); ctx.path = optparse_get_str (p, "path", "eventlog"); ctx.p = p; entry_format_parse_options (p, &ctx.e); @@ -2656,7 +2629,7 @@ int cmd_wait_event (optparse_t *p, int argc, char **argv) optparse_print_usage (p); exit (1); } - ctx.id = parse_arg_unsigned (argv[optindex++], "jobid"); + ctx.id = parse_jobid (argv[optindex++]); ctx.p = p; ctx.wait_event = argv[optindex++]; ctx.timeout = optparse_get_duration (p, "timeout", -1.0); @@ -2734,7 +2707,7 @@ int cmd_info (optparse_t *p, int argc, char **argv) exit (1); } - ctx.id = parse_arg_unsigned (argv[optindex++], "jobid"); + ctx.id = parse_jobid (argv[optindex++]); if (!(ctx.keys = json_array ())) log_msg_exit ("json_array"); @@ -2800,7 +2773,7 @@ int cmd_wait (optparse_t *p, int argc, char **argv) exit (1); } if (optindex < argc) { - id = parse_arg_unsigned (argv[optindex++], "jobid"); + id = parse_jobid (argv[optindex++]); if (optparse_hasopt (p, "all")) log_err_exit ("jobid not supported with --all"); } diff --git a/src/cmd/flux-jobs.py b/src/cmd/flux-jobs.py index c283a47d9cbd..0ea4c7e840f3 100755 --- a/src/cmd/flux-jobs.py +++ b/src/cmd/flux-jobs.py @@ -22,11 +22,11 @@ import json from datetime import datetime, timedelta -import flux.job import flux.constants import flux.util from flux.core.inner import raw from flux.memoized_property import memoized_property +from flux.job import JobID LOGGER = logging.getLogger("flux-jobs") @@ -129,6 +129,9 @@ def __init__(self, info_resp): combined_dict = self.defaults.copy() combined_dict.update(info_resp) + # Cast jobid to JobID + combined_dict["id"] = JobID(combined_dict["id"]) + # Rename "state" to "state_id" and "result" to "result_id" # until returned state is a string: if "state" in combined_dict: @@ -305,6 +308,11 @@ def fetch_jobs_flux(args, fields): # Note there is no attr for "id", its always returned fields2attrs = { "id": (), + "id.hex": (), + "id.f58": (), + "id.kvs": (), + "id.words": (), + "id.dothex": (), "userid": ("userid",), "username": ("userid",), "priority": ("priority",), @@ -477,8 +485,8 @@ def parse_args(): ) parser.add_argument( "jobids", - type=int, metavar="JOBID", + type=JobID, nargs="*", help="Limit output to specific Job IDs", ) @@ -549,6 +557,11 @@ def get_field(self, field_name, args, kwargs): # List of legal format fields and their header names headings = { "id": "JOBID", + "id.hex": "JOBID", + "id.f58": "JOBID", + "id.kvs": "JOBID", + "id.words": "JOBID", + "id.dothex": "JOBID", "userid": "UID", "username": "USER", "priority": "PRI", @@ -571,7 +584,7 @@ def get_field(self, field_name, args, kwargs): "t_inactive": "T_INACTIVE", "runtime": "RUNTIME", "status": "STATUS", - "status_abbrev": "STATUS", + "status_abbrev": "ST", "exception.occurred": "EXCEPTION-OCCURRED", "exception.severity": "EXCEPTION-SEVERITY", "exception.type": "EXCEPTION-TYPE", @@ -636,7 +649,7 @@ def main(): fmt = args.format else: fmt = ( - "{id:>18} {username:<8.8} {name:<10.10} {status_abbrev:>6.6} " + "{id.f58:>12} {username:<8.8} {name:<10.10} {status_abbrev:>2.2} " "{ntasks:>6} {nnodes:>6h} {runtime!F:>8h} " "{ranks:h}" ) diff --git a/src/common/libjob/job.c b/src/common/libjob/job.c index 173c6f8058e0..0b0169dcb98f 100644 --- a/src/common/libjob/job.c +++ b/src/common/libjob/job.c @@ -13,6 +13,7 @@ #endif #include #include +#include #include #if HAVE_FLUX_SECURITY #include @@ -558,6 +559,85 @@ int flux_job_strtoresult (const char *s, flux_job_result_t *result) return -1; } +int flux_job_id_parse (const char *s, flux_jobid_t *idp) +{ + int len; + const char *p = s; + if (s == NULL + || idp == NULL + || (len = strlen (s)) == 0) { + errno = EINVAL; + return -1; + } + /* Remove leading whitespace + */ + while (isspace(*p)) + p++; + /* Ignore any `job.` prefix. This allows a "kvs" encoding + * created by flux_job_id_encode(3) to properly decode. + */ + if (strncmp (p, "job.", 4) == 0) + p += 4; + return fluid_parse (p, idp); +} + +int flux_job_id_encode (flux_jobid_t id, + const char *type, + char *buf, + size_t bufsz) +{ + fluid_string_type_t t; + if (buf == NULL) { + errno = EINVAL; + return -1; + } + if (type == NULL || strcasecmp (type, "dec") == 0) { + int len = snprintf (buf, bufsz, "%ju", (uintmax_t) id); + if (len >= bufsz) { + errno = ENOSPC; + return -1; + } + return 0; + } + if (strcasecmp (type, "hex") == 0) { + int len = snprintf (buf, bufsz, "0x%jx", (uintmax_t) id); + if (len >= bufsz) { + errno = ENOSPC; + return -1; + } + return 0; + } + + /* The following encodings all use fluid_encode(3). + */ + if (strcasecmp (type, "kvs") == 0) { + /* kvs: prepend "job." to "dothex" encoding. + */ + int len = snprintf (buf, bufsz, "job."); + if (len >= bufsz) { + errno = ENOSPC; + return -1; + } + buf += len; + bufsz -= len; + type = "dothex"; + } + if (strcasecmp (type, "dothex") == 0) + t = FLUID_STRING_DOTHEX; + else if (strcasecmp (type, "words") == 0) + t = FLUID_STRING_MNEMONIC; + else if (strcasecmp (type, "f58") == 0) + t = FLUID_STRING_F58; + else { + /* Return EPROTO for invalid type to differentiate from + * other invalid arguments. + */ + errno = EPROTO; + return -1; + } + return fluid_encode (buf, bufsz, id, t); +} + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libjob/job.h b/src/common/libjob/job.h index e9f40dbe20d1..70551321846d 100644 --- a/src/common/libjob/job.h +++ b/src/common/libjob/job.h @@ -63,6 +63,22 @@ typedef enum { typedef uint64_t flux_jobid_t; +/* Parse a jobid from NULL-teminated string 's' in any supported encoding. + * Returns 0 on success, -1 on failure. + */ +int flux_job_id_parse (const char *s, flux_jobid_t *id); + +/* Encode a jobid into encoding "type", writing the result to buffer + * buf of size bufsz. + * Supported encoding types include: + * "dec", "hex", "kvs", "dothex", "words", or "f58". + * Returns 0 on success, -1 on failure with errno set: + * EPROTO: Invalid encoding type + * EINVAL: Invalid other argument + */ +int flux_job_id_encode (flux_jobid_t id, const char *type, + char *buf, size_t bufsz); + const char *flux_job_statetostr (flux_job_state_t state, bool single_char); int flux_job_strtostate (const char *s, flux_job_state_t *state); diff --git a/src/common/libjob/test/job.c b/src/common/libjob/test/job.c index 92b9656ca8fb..5814db064bc0 100644 --- a/src/common/libjob/test/job.c +++ b/src/common/libjob/test/job.c @@ -317,6 +317,82 @@ void check_kvs_namespace (void) "flux_job_kvs_namespace returns EINVAL on invalid buffer"); } +struct jobid_parse_test { + const char *type; + flux_jobid_t id; + const char *string; +}; + +struct jobid_parse_test jobid_parse_tests[] = { + { "dec", 0, "0" }, + { "hex", 0, "0x0" }, + { "dothex", 0, "0000.0000.0000.0000" }, + { "kvs", 0, "job.0000.0000.0000.0000" }, + { "words", 0, "academy-academy-academy--academy-academy-academy" }, + { "f58", 0, "ƒ1" }, + + { "dec", 1, "1" }, + { "hex", 1, "0x1" }, + { "dothex", 1, "0000.0000.0000.0001" }, + { "kvs", 1, "job.0000.0000.0000.0001" }, + { "words", 1, "acrobat-academy-academy--academy-academy-academy" }, + { "f58", 1, "ƒ2" }, + + { "dec", 65535, "65535" }, + { "hex", 65535, "0xffff" }, + { "dothex", 65535, "0000.0000.0000.ffff" }, + { "kvs", 65535, "job.0000.0000.0000.ffff" }, + { "words", 65535, "nevada-archive-academy--academy-academy-academy" }, + { "f58", 65535, "ƒLUv" }, + + { "dec", 6787342413402046, "6787342413402046" }, + { "hex", 6787342413402046, "0x181d0d4d850fbe" }, + { "dothex", 6787342413402046, "0018.1d0d.4d85.0fbe" }, + { "kvs", 6787342413402046, "job.0018.1d0d.4d85.0fbe" }, + { "words", 6787342413402046, "cake-plume-nepal--neuron-pencil-academy" }, + { "f58", 6787342413402046, "ƒuzzybunny" }, + + { NULL, 0, NULL } +}; + +void check_jobid_parse_encode (void) +{ + char buf[1024]; + flux_jobid_t jobid; + struct jobid_parse_test *tp = jobid_parse_tests; + while (tp->type != NULL) { + memset (buf, 0, sizeof (buf)); + ok (flux_job_id_encode (tp->id, tp->type, buf, sizeof (buf)) == 0, + "flux_job_id_encode (%ju, %s) == 0", (uintmax_t) tp->id, tp->type); + is (buf, tp->string, + "flux_job_id_encode() got %s", buf); + ok (flux_job_id_parse (buf, &jobid) == 0, + "flux_job_id_parse() of result works: %s", strerror (errno)); + ok (jobid == tp->id, + "flux_job_id_parse() returned correct id"); + tp++; + } + + ok (flux_job_id_encode (1234, NULL, buf, sizeof (buf)) == 0, + "flux_job_id_encode() with NULL type works"); + is (buf, "1234", + "flux_job_id_encode() encodes to decimal by default"); + + ok (flux_job_id_parse (" 1234 ", &jobid) == 0, + "flux_job_id_parse works with leading whitespace"); + ok (jobid == 1234, + "flux_job_id_parse got expected result"); + + ok (flux_job_id_encode (1234, NULL, NULL, 33) < 0 && errno == EINVAL, + "flux_job_id_encode with NULL buffer returns EINVAL"); + ok (flux_job_id_encode (1234, "dec", buf, 4) < 0 && errno == ENOSPC, + "flux_job_id_encode with too small buffer returns ENOSPC"); + ok (flux_job_id_encode (1234, "dothex", buf, 19) < 0 && errno == ENOSPC, + "flux_job_id_encode with too small buffer returns ENOSPC"); + ok (flux_job_id_encode (1234, "foo", buf, 1024) < 0 && errno == EPROTO, + "flux_job_id_encode with unknown encode type returns EPROTO"); +} + int main (int argc, char *argv[]) { @@ -332,6 +408,8 @@ int main (int argc, char *argv[]) check_kvs_namespace (); + check_jobid_parse_encode (); + done_testing (); return 0; } diff --git a/src/common/libutil/fluid.c b/src/common/libutil/fluid.c index 49e40bf8fc2a..b2baae75746d 100644 --- a/src/common/libutil/fluid.c +++ b/src/common/libutil/fluid.c @@ -18,15 +18,66 @@ #include #include #include +#include +#include +#include #include "fluid.h" #include "mnemonic.h" + /* fluid: [ts:40 id:14 seq:10] */ static const int bits_per_ts = 40; static const int bits_per_id = 14; static const int bits_per_seq = 10; +/* Max base58 string length for F58 encoding */ +#define MAX_B58_STRLEN 12 + +static const char f58_prefix[] = "ƒ"; +static const char f58_alt_prefix[] = "f"; + +/* b58digits_map courtesy of libbase58: + * + * https://github.com/bitcoin/libbase58.git + * + +Copyright (c) 2014 Luke Dashjr + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +*/ + +static const int8_t b58digits_map[] = { + -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, + -1,-1,-1,-1,-1,-1,-1,-1, -1,-1,-1,-1,-1,-1,-1,-1, + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8,-1,-1,-1,-1,-1,-1, + -1, 9,10,11,12,13,14,15, 16,-1,17,18,19,20,21,-1, + 22,23,24,25,26,27,28,29, 30,31,32,-1,-1,-1,-1,-1, + -1,33,34,35,36,37,38,39, 40,41,42,43,-1,44,45,46, + 47,48,49,50,51,52,53,54, 55,56,57,-1,-1,-1,-1,-1, +}; + +static const char b58digits[] = + "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"; + static int current_ds (uint64_t *ds) { struct timespec ts; @@ -97,6 +148,120 @@ int fluid_generate (struct fluid_generator *gen, fluid_t *fluid) return 0; } +/* F58 encoding. + */ +/* Compute base58 encoding of id in *reverse* + * Return number of characters written into buf + */ +static int b58revenc (char *buf, int bufsz, fluid_t id) +{ + int index = 0; + memset (buf, 0, bufsz); + if (id == 0) { + buf[0] = b58digits[0]; + return 1; + } + while (id > 0) { + int rem = id % 58; + buf[index++] = b58digits[rem]; + id = id / 58; + } + return index; +} + +static int fluid_f58_encode (char *buf, int bufsz, fluid_t id) +{ + int count; + char b58reverse[13]; + int index = 0; + if (buf == NULL || bufsz <= 0) { + errno = EINVAL; + return -1; + } + if (bufsz <= strlen (f58_prefix) + 1) { + errno = EOVERFLOW; + return -1; + } + + if ((count = b58revenc (b58reverse, sizeof (b58reverse), id)) < 0) { + errno = EINVAL; + return -1; + } + + /* Copy prefix to buf and zero remaining bytes */ + (void) strncpy (buf, f58_prefix, bufsz); + index = strlen (buf); + + if (index + count + 1 > bufsz) { + errno = EOVERFLOW; + return -1; + } + + for (int i = count - 1; i >= 0; i--) + buf[index++] = b58reverse[i]; + + return 0; +} + +static int b58decode (const char *str, uint64_t *idp) +{ + int64_t id = 0; + int64_t scale = 1; + int len = strlen (str); + if (len == 0) { + errno = EINVAL; + return -1; + } + for (int i = len - 1; i >= 0; i--) { + int8_t c = b58digits_map[(int8_t)str[i]]; + if (c == -1) { + /* invalid base58 digit */ + errno = EINVAL; + return -1; + } + id += c * scale; + scale *= 58; + } + *idp = id; + return 0; +} + +static int fluid_is_f58 (const char *str) +{ + int len = 0; + if (str == NULL || str[0] == '\0') + return 0; + len = strlen (f58_prefix); + if (strncmp (str, f58_prefix, len) == 0) + return len; + len = strlen (f58_alt_prefix); + if (strncmp (str, f58_alt_prefix, len) == 0) + return len; + return 0; +} + +static int fluid_f58_decode (fluid_t *idptr, const char *str) +{ + int prefix = 0; + const char *b58str = NULL; + + if (idptr == NULL || str == NULL) { + errno = EINVAL; + return -1; + } + if ((prefix = fluid_is_f58 (str)) == 0) { + /* no prefix match, not valid f58 */ + errno = EINVAL; + return -1; + } + b58str = str+prefix; + if (strlen (b58str) > MAX_B58_STRLEN) { + errno = EINVAL; + return -1; + } + return b58decode (str+prefix, idptr); +} + static int fluid_decode_dothex (const char *s, fluid_t *fluid) { int i; @@ -141,6 +306,10 @@ int fluid_encode (char *buf, int bufsz, fluid_t fluid, buf, bufsz, MN_FDEFAULT) != MN_OK) return -1; break; + case FLUID_STRING_F58: + if (fluid_f58_encode (buf, bufsz, fluid) < 0) + return -1; + break; } return 0; } @@ -178,11 +347,17 @@ int fluid_decode (const char *s, fluid_t *fluidp, fluid_string_type_t type) * Also, 's' is not modified so it is safe to cast away const. */ rc = mn_decode ((char *)s, (void *)&fluid, sizeof (fluid_t)); - if (rc != 8) + if (rc != 8) { + errno = EINVAL; + return -1; + } + break; + case FLUID_STRING_F58: + if (fluid_f58_decode (&fluid, s) < 0) return -1; break; - default: + errno = EINVAL; return -1; } if (fluid_validate (fluid) < 0) @@ -191,6 +366,81 @@ int fluid_decode (const char *s, fluid_t *fluidp, fluid_string_type_t type) return 0; } +static bool fluid_is_dothex (const char *s) +{ + return (strchr (s, '.') != NULL); +} + +static bool fluid_is_words (const char *s) +{ + return (strchr (s, '-') != NULL); +} + +fluid_string_type_t fluid_string_detect_type (const char *s) +{ + /* N.B.: An F58 encoded FLUID may start with 'f', which also could + * be true for dothex or words representations. Therefore, always + * check for these encodings first, since F58 must not have '.' + * or '-' characters, which distinguish dothex and mnemonic. + */ + if (fluid_is_dothex (s)) + return FLUID_STRING_DOTHEX; + if (fluid_is_words (s)) + return FLUID_STRING_MNEMONIC; + if (fluid_is_f58 (s) > 0) + return FLUID_STRING_F58; + return 0; +} + +static bool is_trailing_space (const char *p) +{ + while (*p != '\0' && isspace (*p)) + p++; + return (*p == '\0'); +} + +int fluid_parse (const char *s, fluid_t *fluidp) +{ + int base = 10; + unsigned long long l; + char *endptr; + fluid_string_type_t type; + + if (s == NULL || s[0] == '\0') { + errno = EINVAL; + return -1; + } + + /* Skip leading whitepsace + */ + while (*s != '\0' && isspace (*s)) + s++; + + if ((type = fluid_string_detect_type (s)) != 0) + return fluid_decode (s, fluidp, type); + + /* O/w, FLUID encoded as an integer, either base16 (prefix="0x") + * or base10 (no prefix). + */ + if (strncmp (s, "0x", 2) == 0) + base = 16; + errno = 0; + l = strtoull (s, &endptr, base); + if (errno != 0) + return -1; + /* Ignore trailing whitespace */ + if (!is_trailing_space(endptr)) { + errno = EINVAL; + return -1; + } + *fluidp = l; + + if (fluid_validate (*fluidp) < 0) + return -1; + + return 0; +} + /* * vi:tabstop=4 shiftwidth=4 expandtab */ diff --git a/src/common/libutil/fluid.h b/src/common/libutil/fluid.h index bf196699ae82..5e0ff06a761d 100644 --- a/src/common/libutil/fluid.h +++ b/src/common/libutil/fluid.h @@ -22,6 +22,7 @@ typedef enum { FLUID_STRING_DOTHEX = 1, // x.x.x.x FLUID_STRING_MNEMONIC = 2, // mnemonicode x-x-x--x-x-x + FLUID_STRING_F58 = 3, // FLUID base58 enc: ƒXXXX or fXXXX } fluid_string_type_t; struct fluid_generator { @@ -68,6 +69,20 @@ int fluid_encode (char *buf, int bufsz, fluid_t fluid, int fluid_decode (const char *s, fluid_t *fluid, fluid_string_type_t type); +/* Attempt to detect the string type of encoded FLUID in `s`. + * returns the string type or 0 if not one the defined encodings above. + * (FLUID may still be encoded as integer in decimal or hex) + */ +fluid_string_type_t fluid_string_detect_type (const char *s); + +/* Convert NULL-terminated string 's' to 'fluid' by auto-detecting + * the encoding in 's'. + * Supported encodings include any fluid_string_type_t, or an integer + * in decimal or hexidecimal prefixed with "0x". + * Return 0 on success, -1 on failure. + */ +int fluid_parse (const char *s, fluid_t *fluid); + #endif /* !_UTIL_FLUID_H */ /* diff --git a/src/common/libutil/test/fluid.c b/src/common/libutil/test/fluid.c index d25d477f8c63..7b579db49307 100644 --- a/src/common/libutil/test/fluid.c +++ b/src/common/libutil/test/fluid.c @@ -8,10 +8,188 @@ * SPDX-License-Identifier: LGPL-3.0 \************************************************************/ +#include + #include "src/common/libtap/tap.h" #include "src/common/libutil/fluid.h" -int main (int argc, char *argv[]) +struct f58_test { + fluid_t id; + const char *f58; +}; + +struct f58_test f58_tests [] = { + { 0, "ƒ1" }, + { 1, "ƒ2" }, + { 57, "ƒz" }, + { 1234, "ƒNH" }, + { 1888, "ƒZZ" }, + { 3363, "ƒzz" }, + { 3364, "ƒ211" }, + { 4369, "ƒ2JL" }, + { 65535, "ƒLUv" }, + { 4294967295, "ƒ7YXq9G" }, + { 633528662, "ƒxyzzy" }, + { 6731191091817518LL, "ƒuZZybuNNy" }, + { 18446744073709551614UL, "ƒjpXCZedGfVP" }, + { 18446744073709551615UL, "ƒjpXCZedGfVQ" }, + { 0, NULL }, +}; + +struct f58_test f58_alt_tests [] = { + { 0, "f1" }, + { 0, "f111" }, + { 1, "f2" }, + { 57, "fz" }, + { 1234, "fNH" }, + { 1888, "fZZ" }, + { 3363, "fzz" }, + { 3364, "f211" }, + { 4369, "f2JL" }, + { 65535, "fLUv" }, + { 4294967295, "f7YXq9G" }, + { 633528662, "fxyzzy" }, + { 6731191091817518LL, "fuZZybuNNy" }, + { 18446744073709551614UL, "fjpXCZedGfVP" }, + { 18446744073709551615UL, "fjpXCZedGfVQ" }, + { 0, NULL }, +}; + +void test_f58 (void) +{ + fluid_string_type_t type = FLUID_STRING_F58; + char buf[16]; + fluid_t id; + struct f58_test *tp = f58_tests; + while (tp->f58 != NULL) { + ok (fluid_encode (buf, sizeof(buf), tp->id, type) == 0, + "f58_encode (%ju)", tp->id); + is (buf, tp->f58, + "f58_encode %ju -> %s", tp->id, buf); + ok (fluid_decode (tp->f58, &id, type) == 0, + "f58_decode (%s)", tp->f58); + ok (id == tp->id, + "%s -> %ju", tp->f58, (uintmax_t) id); + tp++; + } + tp = f58_alt_tests; + while (tp->f58 != NULL) { + ok (fluid_decode (tp->f58, &id, type) == 0, + "f58_decode (%s)", tp->f58); + ok (id == tp->id, + "%s -> %ju", tp->f58, (uintmax_t) id); + tp++; + } + + ok (fluid_encode (buf, 1, 1, type) < 0 && errno == EOVERFLOW, + "fluid_encode (buf, 1, 1, F58) returns EOVERFLOW"); + ok (fluid_encode (buf, 5, 65535, type) < 0 && errno == EOVERFLOW, + "fluid_encode (buf, 5, 65535, F58) returns EOVERFLOW"); + + ok (fluid_decode ("1234", &id, type) < 0 && errno == EINVAL, + "fluid_decode ('aaa', FLUID_STRING_F58) returns EINVAL"); + ok (fluid_decode ("aaa", &id, type) < 0 && errno == EINVAL, + "fluid_decode ('aaa', FLUID_STRING_F58) returns EINVAL"); + ok (fluid_decode ("f", &id, type) < 0 && errno == EINVAL, + "fluid_decode ('f', FLUID_STRING_F58) returns EINVAL"); + ok (fluid_decode ("flux", &id, type) < 0 && errno == EINVAL, + "fluid_decode ('flux', FLUID_STRING_F58) returns EINVAL"); + ok (fluid_decode ("f1230", &id, type) < 0 && errno == EINVAL, + "fluid_decode ('f1230', FLUID_STRING_F58) returns EINVAL"); + ok (fluid_decode ("x1", &id, type) < 0 && errno == EINVAL, + "fluid_decode ('x1', FLUID_STRING_F58) returns EINVAL"); +} + +struct fluid_parse_test { + fluid_t id; + const char *input; +}; + +struct fluid_parse_test fluid_parse_tests [] = { + { 0, "ƒ1" }, + { 1, "ƒ2" }, + { 57, "ƒz" }, + { 1234, "ƒNH" }, + { 1888, "ƒZZ" }, + { 3363, "ƒzz" }, + { 3364, "ƒ211" }, + { 4369, "ƒ2JL" }, + { 65535, "ƒLUv" }, + { 4294967295, "ƒ7YXq9G" }, + { 633528662, "ƒxyzzy" }, + { 6731191091817518LL, "ƒuZZybuNNy" }, + { 18446744073709551614UL, "ƒjpXCZedGfVP" }, + { 18446744073709551615UL, "ƒjpXCZedGfVQ" }, + { 0, "f1" }, + { 1, "f2" }, + { 4294967295, "f7YXq9G" }, + { 633528662, "fxyzzy" }, + { 18446744073709551614UL, "fjpXCZedGfVP" }, + { 18446744073709551615UL, "fjpXCZedGfVQ" }, + { 1234, "1234" }, + { 1888, "1888" }, + { 3363, "3363" }, + { 3364, "3364" }, + { 4369, "4369" }, + { 6731191091817518LL, "6731191091817518" }, + { 18446744073709551614UL, "18446744073709551614" }, + { 18446744073709551615UL, "18446744073709551615" }, + { 0, "0x0" }, + { 1, "0x1" }, + { 57, "0x39" }, + { 1234, "0x4d2" }, + { 1888, "0x760" }, + { 3363, "0xd23" }, + { 4369, "0x1111" }, + { 65535, "0xffff" }, + { 4294967295, "0xffffffff" }, + { 633528662, "0x25c2e156" }, + { 6731191091817518LL, "0x17e9fb8df16c2e" }, + { 18446744073709551615UL, "0xffffffffffffffff" }, + { 0, "0.0.0.0" }, + { 1, "0000.0000.0000.0001" }, + { 57, "0.0.0.0039" }, + { 1234, "0000.0000.0000.04d2" }, + { 1888, "0000.0000.0000.0760" }, + { 4369, "0000.0000.0000.1111" }, + { 65535, "0.0.0.ffff" }, + { 4294967295, "0000.0000.ffff.ffff" }, + { 18446744073709551615UL, "ffff.ffff.ffff.ffff" }, + { 0, NULL }, +}; + +static void test_fluid_parse (void) +{ + fluid_t id; + struct fluid_parse_test *tp = fluid_parse_tests; + while (tp->input != NULL) { + id = 0; + ok (fluid_parse (tp->input, &id) == 0, + "fluid_parse (%s) works", tp->input); + ok (id == tp->id, + "%s -> %ju", tp->input, (uintmax_t) id); + tp++; + } + + ok (fluid_parse (" 0xffff ", &id) == 0, + "flux_parse() works with leading/trailing whitespace"); + ok (id == 65535, + "flux_parse with whitespace works"); + + id = 0; + ok (fluid_parse (NULL, &id) < 0 && errno == EINVAL, + "fluid_parse returns EINVAL for with NULL string"); + ok (fluid_parse ("", &id) < 0 && errno == EINVAL, + "fluid_parse returns EINVAL for with empty string"); + ok (fluid_parse ("boo", &id) < 0 && errno == EINVAL, + "fluid_parse returns EINVAL for 'boo'"); + ok (fluid_parse ("f", &id) < 0 && errno == EINVAL, + "fluid_parse returns EINVAL for 'f'"); + ok (fluid_parse ("-1", &id) < 0 && errno == EINVAL, + "fluid_parse returns EINVAL for '-1'"); +} + +void test_basic (void) { struct fluid_generator gen; fluid_t id, id2; @@ -21,8 +199,6 @@ int main (int argc, char *argv[]) int encode_errors; int decode_errors; - plan (NO_PLAN); - ok (fluid_init (&gen, 0, 0) == 0, "fluid_init id=0 timestamp=0 works"); @@ -136,6 +312,15 @@ int main (int argc, char *argv[]) "fluid_decode type=MNEMONIC fails on input=bogus"); ok (fluid_decode ("a-a-a--a-a-a", &id, FLUID_STRING_MNEMONIC) < 0, "fluid_decode type=MNEMONIC fails on unknown words xx-xx-xx--xx-xx-xx"); +} + +int main (int argc, char *argv[]) +{ + plan (NO_PLAN); + + test_basic (); + test_f58 (); + test_fluid_parse (); done_testing (); return 0; diff --git a/t/python/t0010-job.py b/t/python/t0010-job.py index b567718c77b5..015f02b66292 100755 --- a/t/python/t0010-job.py +++ b/t/python/t0010-job.py @@ -365,6 +365,66 @@ def test_23_from_nest_command(self): jobid = job.submit(self.fh, JobspecV1.from_nest_command(["sleep", "0"])) self.assertGreater(jobid, 0) + def test_24_jobid(self): + """Test JobID class""" + parse_tests = [ + { + "int": 0, + "dec": "0", + "hex": "0x0", + "dothex": "0000.0000.0000.0000", + "kvs": "job.0000.0000.0000.0000", + "f58": "ƒ1", + "words": "academy-academy-academy--academy-academy-academy", + }, + { + "int": 1, + "dec": "1", + "hex": "0x1", + "dothex": "0000.0000.0000.0001", + "kvs": "job.0000.0000.0000.0001", + "f58": "ƒ2", + "words": "acrobat-academy-academy--academy-academy-academy", + }, + { + "int": 65535, + "dec": "65535", + "hex": "0xffff", + "dothex": "0000.0000.0000.ffff", + "kvs": "job.0000.0000.0000.ffff", + "f58": "ƒLUv", + "words": "nevada-archive-academy--academy-academy-academy", + }, + { + "int": 6787342413402046, + "dec": "6787342413402046", + "hex": "0x181d0d4d850fbe", + "dothex": "0018.1d0d.4d85.0fbe", + "kvs": "job.0018.1d0d.4d85.0fbe", + "f58": "ƒuzzybunny", + "words": "cake-plume-nepal--neuron-pencil-academy", + }, + { + "int": 18446744073709551614, + "dec": "18446744073709551614", + "hex": "0xfffffffffffffffe", + "dothex": "ffff.ffff.ffff.fffe", + "kvs": "job.ffff.ffff.ffff.fffe", + "f58": "ƒjpXCZedGfVP", + "words": "mustang-analyze-verbal--natural-analyze-verbal", + }, + ] + for test in parse_tests: + for key in test: + jobid_int = test["int"] + jobid = job.JobID(test[key]) + self.assertEqual(jobid, jobid_int) + jobid_repr = repr(jobid) + self.assertEqual(jobid_repr, f"JobID({jobid_int})") + if key not in ["int", "dec"]: + # Ensure encode back to same type works + self.assertEqual(getattr(jobid, key), test[key]) + if __name__ == "__main__": from subflux import rerun_under_flux diff --git a/t/t2201-job-cmd.t b/t/t2201-job-cmd.t index 5a7c23bd32b0..4fd87c1a9675 100755 --- a/t/t2201-job-cmd.t +++ b/t/t2201-job-cmd.t @@ -9,14 +9,20 @@ fi # 2^64 - 1 MAXJOBID_DEC=18446744073709551615 +MAXJOBID_HEX="0xffffffffffffffff" MAXJOBID_KVS="job.ffff.ffff.ffff.ffff" -MAXJOBID_HEX="ffff.ffff.ffff.ffff" +MAXJOBID_DOTHEX="ffff.ffff.ffff.ffff" MAXJOBID_WORDS="natural-analyze-verbal--natural-analyze-verbal" +MAXJOBID_F58="ƒjpXCZedGfVQ" +MAXJOBIDS_LIST="$MAXJOBID_DEC $MAXJOBID_HEX $MAXJOBID_KVS $MAXJOBID_DOTHEX $MAXJOBID_WORDS $MAXJOBID_F58" MINJOBID_DEC=0 +MINJOBID_HEX="0x0" MINJOBID_KVS="job.0000.0000.0000.0000" -MINJOBID_HEX="0000.0000.0000.0000" +MINJOBID_DOTHEX="0000.0000.0000.0000" MINJOBID_WORDS="academy-academy-academy--academy-academy-academy" +MINJOBID_F58="ƒ1" +MINJOBIDS_LIST="$MINJOBID_DEC $MINJOBID_HEX $MINJOBID_KVS $MINJOBID_DOTHEX $MINJOBID_WORDS $MINJOBID_F58" test_under_flux 1 job @@ -83,7 +89,7 @@ test_expect_success 'flux-job: can submit jobspec on stdin without -' ' flux job submit $jobid" && + test "$jobid" = "$MAXJOBID_DEC" + done && + for min in $MINJOBIDS_LIST; do + jobid=$(flux job id $min) && + test_debug "echo flux jobid $min -> $jobid" && + test "$jobid" = "$MINJOBID_DEC" + done ' test_expect_success 'flux-job: id --to=dec works' ' @@ -154,22 +144,34 @@ test_expect_success 'flux-job: id --to=hex works' ' test "$jobid" = "$MINJOBID_HEX" ' -test_expect_success 'flux-job: id --from=kvs fails on bad input' ' - test_must_fail flux job id --from=kvs badstring && - test_must_fail flux job id --from=kvs \ - job.0000.0000 && - test_must_fail flux job id --from=kvs \ - job.0000.0000.0000.000P +test_expect_success 'flux-job: id --to=dothex works' ' + jobid=$(flux job id --to=dothex $MAXJOBID_DEC) && + test "$jobid" = "$MAXJOBID_DOTHEX" && + jobid=$(flux job id --to=dothex $MINJOBID_DEC) && + test "$jobid" = "$MINJOBID_DOTHEX" +' + +test_expect_success 'flux-job: id --to=f58 works' ' + jobid=$(flux job id --to=f58 $MAXJOBID_DEC) && + test "$jobid" = "$MAXJOBID_F58" && + jobid=$(flux job id --to=f58 $MINJOBID_DEC) && + test "$jobid" = "$MINJOBID_F58" +' + +test_expect_success 'flux-job: id fails on bad input' ' + test_must_fail flux job id badstring && + test_must_fail flux job id job.0000.0000 && + test_must_fail flux job id job.0000.0000.0000.000P ' -test_expect_success 'flux-job: id --from=dec fails on bad input' ' - test_must_fail flux job id --from=dec 42plusbad && - test_must_fail flux job id --from=dec meep && - test_must_fail flux job id --from=dec 18446744073709551616 +test_expect_success 'flux-job: id fails on bad input' ' + test_must_fail flux job id 42plusbad && + test_must_fail flux job id meep && + test_must_fail flux job id 18446744073709551616 ' -test_expect_success 'flux-job: id --from=words fails on bad input' ' - test_must_fail flux job id --from=words badwords +test_expect_success 'flux-job: id fails on bad words input' ' + test_must_fail flux job id bad-words ' test_expect_success 'flux-job: priority fails with bad FLUX_URI' ' diff --git a/t/t2800-jobs-cmd.t b/t/t2800-jobs-cmd.t index f48e9c765beb..49b1624c4b78 100755 --- a/t/t2800-jobs-cmd.t +++ b/t/t2800-jobs-cmd.t @@ -293,6 +293,16 @@ test_expect_success 'flux-jobs specific IDs works' ' done ' +test_expect_success 'flux jobs can take specific IDs in any form' ' + id=$(head -1 run.ids) && + for f in f58 hex dothex kvs words; do + flux job id --to=${f} ${id} + done > ids.specific.list && + flux jobs -no {id} $(cat ids.specific.list) > ids.specific.out && + for i in $(seq 1 5); do echo $id >>ids.specific.expected; done && + test_cmp ids.specific.expected ids.specific.out +' + test_expect_success 'flux-jobs error on bad IDs' ' flux jobs --suppress-header 0 1 2 2> ids.err && count=`grep -i unknown ids.err | wc -l` && @@ -327,6 +337,21 @@ test_expect_success 'flux-jobs --format={id} works' ' test_cmp idsI.out inactive.ids ' +test_expect_success 'flux-jobs --format={id.f58},{id.hex},{id.dothex},{id.words} works' ' + flux jobs -ano {id},{id.f58},{id.hex},{id.kvs},{id.dothex},{id.words} \ + | sort -n > ids.XX.out && + for id in $(cat all.ids); do + printf "%s,%s,%s,%s,%s,%s\n" \ + $id \ + $(flux job id --to=f58 $id) \ + $(flux job id --to=hex $id) \ + $(flux job id --to=kvs $id) \ + $(flux job id --to=dothex $id) \ + $(flux job id --to=words $id) + done | sort -n > ids.XX.expected && + test_cmp ids.XX.expected ids.XX.out +' + test_expect_success 'flux-jobs --format={userid},{username} works' ' flux jobs --suppress-header -a --format="{userid},{username}" > user.out && id=`id -u` && @@ -624,6 +649,10 @@ test_expect_success 'flux-jobs --format={expiration!D:h},{t_remaining!H:h} works test_expect_success 'flux-jobs: header included with all custom formats' ' cat <<-EOF >headers.expected && id==JOBID + id.f58==JOBID + id.hex==JOBID + id.dothex==JOBID + id.words==JOBID userid==UID username==USER priority==PRI @@ -658,6 +687,8 @@ test_expect_success 'flux-jobs: header included with all custom formats' ' t_inactive!d==T_INACTIVE t_cleanup!D==T_CLEANUP t_run!F==T_RUN + status==STATUS + status_abbrev==ST EOF sed "s/\(.*\)==.*/\1=={\1}/" headers.expected > headers.fmt && flux jobs --from-stdin --format="$(cat headers.fmt)" \