Merge pull request #3206 from grondo/job-output-format

shell: render output file template {{id}} as F58 by default, allow other id encodings

doc/man1/flux-mini.rst

@@ -108,17 +108,20 @@ KVS, where they may be accessed with the ``flux job attach`` command.
 In addition, ``flux-mini run`` processes standard I/O in real time,
 emitting the job's I/O to its stdout and stderr.
 
-**--output=FILENAME**
-   Redirect stdout to the specified FILENAME, bypassing the KVS.
-   The mustache template *{{id}}* is expanded to the numerical Job ID,
-   useful to ensure FILENAME is unique across multiple jobs. For
-   **flux mini batch** the default for FILENAME is *flux-{{id}}.out*.
-   To force output to go to KVS so it is available with ``flux job attach``,
-   set FILENAME to *none* or *kvs*.
-
-**--error=FILENAME**
-   Redirect stderr to the specified FILENAME, bypassing the KVS.
-   The mustache template *{{id}}* is expanded as above.
+**--output=TEMPLATE**
+   Specify the filename *TEMPLATE* for stdout redirection, bypassing
+   the KVS.  *TEMPLATE* may be a mustache template which supports the
+   tags *{{id}}* and *{{jobid}}* which expand to the current jobid
+   in the F58 encoding.  If needed, an alternate encoding can be
+   selected by using a subkey with the name of the desired encoding,
+   e.g. *{{id.dec}}*. Supported encodings include *f58* (the default),
+   *dec*, *hex*, *dothex*, and *words*. For **flux mini batch** the
+   default *TEMPLATE* is *flux-{{id}}.out*. To force output to KVS so it is
+   available with ``flux job attach``, set *TEMPLATE* to *none* or *kvs*.
+
+**--error=TEMPLATE**
+   Redirect stderr to the specified filename *TEMPLATE*, bypassing the KVS.
+   *TEMPLATE* is expanded as described above.
 
 **-l, --label-io**
    Add task rank prefixes to each line of output.

doc/test/spell.en.pws

@@ -515,3 +515,6 @@ builtin
 OMP
 envfile
 regex
+encodings
+dec
+subkey

src/shell/Makefile.am

@@ -83,7 +83,9 @@ flux_shell_SOURCES = \
 	pty.c \
 	batch.c \
 	mpir/mpir.c \
-	mpir/ptrace.c
+	mpir/ptrace.c \
+	mustache.h \
+	mustache.c
 
 flux_shell_LDADD = \
 	$(builddir)/libshell.la \
@@ -108,6 +110,7 @@ EXTRA_DIST = \
 TESTS = \
 	test_jobspec.t \
 	test_plugstack.t \
+	test_mustache.t \
 	mpir/test_rangelist.t \
 	mpir/test_nodelist.t \
 	mpir/test_proctable.t
@@ -188,3 +191,15 @@ mpir_test_proctable_t_LDADD = \
 	$(test_ldadd)
 mpir_test_proctable_t_LDFLAGS = \
 	$(test_ldflags)
+
+test_mustache_t_SOURCES = \
+	mustache.c \
+	mustache.h \
+	test/mustache.c
+test_mustache_t_CPPFLAGS = \
+	$(test_cppflags)
+test_mustache_t_LDADD = \
+	$(builddir)/libshell.la \
+	$(test_ldadd)
+test_mustache_t_LDFLAGS = \
+	$(test_ldflags)

src/shell/log.c

@@ -200,6 +200,27 @@ void flux_shell_log (int level,
     va_end (ap);
 }
 
+/* llog compatible wrapper for flux_shell_log
+ */
+void shell_llog (void *arg,
+                 const char *file,
+                 int line,
+                 const char *func,
+                 const char *subsys,
+                 int level,
+                 const char *fmt,
+                 va_list ap)
+{
+    char buf [4096];
+    int buflen = sizeof (buf);
+    int n = vsnprintf (buf, buflen, fmt, ap);
+    if (n >= buflen) {
+        buf[buflen-1] = '\0';
+        buf[buflen-2] = '+';
+    }
+    flux_shell_log (level, file, line, "%s", buf);
+}
+
 int flux_shell_err (const char *file,
                    int line,
                    int errnum,
@@ -354,6 +375,7 @@ void shell_log_fini (void)
     fclose (logger.fp);
 }
 
+
 /*
  * vi:tabstop=4 shiftwidth=4 expandtab
  */

src/shell/log.h

@@ -32,6 +32,17 @@ void shell_log_set_level (int level);
  */
 void shell_log_set_exception_logged (void);
 
+/*  Shell log function compatible with libutil llog interface
+ */
+void shell_llog (void *arg,
+                 const char *file,
+                 int line,
+                 const char *func,
+                 const char *subsys,
+                 int level,
+                 const char *fmt,
+                 va_list ap);
+
 #endif /* !_SHELL_RC_H */
 
 /* vi: ts=4 sw=4 expandtab

src/shell/mustache.c

@@ -0,0 +1,146 @@
+/************************************************************\
+ * Copyright 2019 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#if HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include <stdio.h>
+#include <string.h>
+
+#include "mustache.h"
+#include "src/common/libutil/llog.h"
+
+struct mustache_renderer {
+    mustache_tag_f tag_f;
+    void *tag_arg;
+
+    mustache_log_f llog;
+    void *llog_data;
+};
+
+void mustache_renderer_destroy (struct mustache_renderer *mr)
+{
+    free (mr);
+}
+
+
+struct mustache_renderer * mustache_renderer_create (mustache_tag_f tagfn,
+                                                     void *arg)
+{
+    struct mustache_renderer *mr = NULL;
+
+    if (!tagfn) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(mr = calloc (1, sizeof (*mr))))
+        return NULL;
+    mr->tag_f = tagfn;
+    mr->tag_arg = arg;
+    return (mr);
+}
+
+void mustache_renderer_set_log (struct mustache_renderer *mr,
+                                mustache_log_f log_f,
+                                void *log_arg)
+{
+    if (mr) {
+        mr->llog = log_f;
+        mr->llog_data = log_arg;
+    }
+}
+
+char *mustache_render (struct mustache_renderer *mr, const char *template)
+{
+    char name [1024];
+    size_t size;
+    char *result = NULL;
+    const char *pos = template;
+    FILE *fp = NULL;
+
+    if (mr == NULL || template == NULL) {
+        errno = EINVAL;
+        return NULL;
+    }
+    if (!(fp = open_memstream (&result, &size))) {
+        llog_error (mr, "open_memstream");
+        goto fail;
+    }
+    for (;;) {
+        int len;
+        char *end;
+
+        /*  Look for opening "{{"
+         */
+        char *start = strstr (pos, "{{");
+        if (start == NULL) {
+            /*  No more mustache tags, put rest of string and finish
+             */
+            if (fputs (pos, fp) < 0) {
+                llog_error (mr, "fputs(%s): %s", pos, strerror (errno));
+                goto fail;
+            }
+            break;
+        }
+        /*  Write any part of template from current position to next tag
+         */
+        if (start > pos && fwrite (pos, start - pos, 1, fp) == 0) {
+            llog_error (mr, "fwrite: %s", strerror (errno));
+            goto fail;
+        }
+        /*  Advance past opening braces
+         */
+        start += 2;
+
+        /*  Find end of template tag
+         */
+        end = strstr (start, "}}");
+        if (end == NULL) {
+            llog_error (mr, "mustache template error at pos=%d",
+                        (int)(start-template));
+            /*  Copy rest of template and exit
+             */
+            pos = start - 2;
+            if (fputs (pos, fp) < 0) {
+                llog_error (mr, "fputs(%s): %s", pos, strerror (errno));
+                goto fail;
+            }
+            break;
+        }
+        /*  Copy mustache tag into 'name' and substitute with callback
+         */
+        len = end - start;
+        memcpy (name, start, len);
+        name[len] = '\0';
+        if ((*mr->tag_f) (fp, name, mr->tag_arg) < 0) {
+            /*  If callback fails, just fail to expand the current mustache tag.
+             */
+            fprintf (fp, "{{%s}}", name);
+        }
+
+        /*  Advance past closing braces '}}' and continue processing
+         */
+        pos = end + 2;
+    }
+    fclose (fp);
+    return result;
+fail:
+    if (fp) {
+        fclose (fp);
+        free (result);
+    }
+    return strdup (template);
+}
+
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */

src/shell/mustache.h

@@ -0,0 +1,60 @@
+/************************************************************\
+ * Copyright 2020 Lawrence Livermore National Security, LLC
+ * (c.f. AUTHORS, NOTICE.LLNS, COPYING)
+ *
+ * This file is part of the Flux resource manager framework.
+ * For details, see https://github.com/flux-framework.
+ *
+ * SPDX-License-Identifier: LGPL-3.0
+\************************************************************/
+
+#ifndef _SHELL_MUSTACHE_H
+#define _SHELL_MUSTACHE_H
+
+#include <stdio.h>
+
+typedef void (*mustache_log_f) (void *arg,
+                                const char *file,
+                                int line,
+                                const char *func,
+                                const char *subsys,
+                                int level,
+                                const char *fmt,
+                                va_list args);
+
+/*  Mustache tag callback function prototype. This function is called
+ *   for any mustache tag 'name' found in the template by the mustache
+ *   template renderer. E.g. if {{foo}} then the tag callback will be
+ *   called with `tag == "foo"`. The function should just write the value
+ *   of `tag` to the FILE stream `fp` if found.
+ *
+ * This function should return < 0 on error, in which case the template
+ *   renderer will copy the unsubstituted mustache tag into the result.
+ */
+typedef int (*mustache_tag_f) (FILE *fp, const char *tag, void *arg);
+
+struct mustache_renderer;
+
+/*  Create a mustache renderer, with mustache tags expanded by tag
+ *   callback `cb`. (See callback definition above).
+ */
+struct mustache_renderer *mustache_renderer_create (mustache_tag_f cb,
+                                                    void *arg);
+
+void mustache_renderer_destroy (struct mustache_renderer *mr);
+
+/*  Set a custom logger for mustache renderer 'mr'.
+ */
+void mustache_renderer_set_log (struct mustache_renderer *mr,
+                                mustache_log_f log,
+                                void *log_data);
+
+/*  Render the mustache template 'template' with renderer 'mr'.
+ */
+char * mustache_render (struct mustache_renderer *mr, const char *template);
+
+#endif /* !_SHELL_MUSTACHE_H */
+
+/*
+ * vi:tabstop=4 shiftwidth=4 expandtab
+ */