libkvs: support KVS_CHECKPOINT_FLAG_CACHE_BYPASS

Problem: The kvs_checkpoint functions checkpoint data to the backing
store through the broker's content-cache.  There is no way to
bypass the cache, similar to libcontent's CONTENT_FLAG_CACHE_BYPASS
flag.

Solution: Support a new KVS_CHECKPOINT_FLAG_CACHE_BYPASS, that will
get/put directly to the backing store.  The flag is very similar to
libcontent's CONTENT_FLAG_CACHE_BYPASS flag.

Fixes flux-framework#4476

src/common/libkvs/kvs_checkpoint.c

@@ -28,19 +28,22 @@ flux_future_t *kvs_checkpoint_commit (flux_t *h,
                                       int flags)
 {
     flux_future_t *f = NULL;
+    const char *topic = "content.checkpoint-put";
+    int valid_flags = KVS_CHECKPOINT_FLAG_CACHE_BYPASS;
 
-    /* no flags supported at the moment */
-    if (!h || !rootref || flags) {
+    if (!h || !rootref || (flags & ~valid_flags)) {
         errno = EINVAL;
         return NULL;
     }
     if (!key)
         key = KVS_DEFAULT_CHECKPOINT;
     if (timestamp == 0)
         timestamp = flux_reactor_now (flux_get_reactor (h));
+    if (flags & KVS_CHECKPOINT_FLAG_CACHE_BYPASS)
+        topic = "content-backing.checkpoint-put";
 
     if (!(f = flux_rpc_pack (h,
-                             "content.checkpoint-put",
+                             topic,
                              0,
                              0,
                              "{s:s s:{s:i s:s s:i s:f}}",
@@ -58,16 +61,20 @@ flux_future_t *kvs_checkpoint_commit (flux_t *h,
 
 flux_future_t *kvs_checkpoint_lookup (flux_t *h, const char *key, int flags)
 {
-    /* no flags supported at the moment */
-    if (!h || flags) {
+    const char *topic = "content.checkpoint-get";
+    int valid_flags = KVS_CHECKPOINT_FLAG_CACHE_BYPASS;
+
+    if (!h || (flags & ~valid_flags)) {
         errno = EINVAL;
         return NULL;
     }
     if (!key)
         key = KVS_DEFAULT_CHECKPOINT;
+    if (flags & KVS_CHECKPOINT_FLAG_CACHE_BYPASS)
+        topic = "content-backing.checkpoint-get";
 
     return flux_rpc_pack (h,
-                          "content.checkpoint-get",
+                          topic,
                           0,
                           0,
                           "{s:s}",

src/common/libkvs/kvs_checkpoint.h

@@ -13,6 +13,11 @@
 
 #include <flux/core.h>
 
+/* flags */
+enum {
+    KVS_CHECKPOINT_FLAG_CACHE_BYPASS = 1,/* request direct to backing store */
+};
+
 #define KVS_DEFAULT_CHECKPOINT "kvs-primary"
 
 /* Calls to kvs_checkpoint_commit() can be racy when the KVS module is