From aac39d267ba8d2e7c6b16f4528d574d428a03e3d Mon Sep 17 00:00:00 2001
From: Albert Chu <chu11@llnl.gov>
Date: Tue, 23 Aug 2022 21:52:28 -0700
Subject: [PATCH] testsuite: add checkpoint tests on rank > 0

Problem: There were no content checkpoint tests on rank > 0
to ensure RPCs are forwarded to rank 0 appropriately.

Solution: Add some non-rank 0 checkpoint tests to t0012-content-sqlite.t,
t0018-content-files.t, and t0024-content-s3.t.
---
 t/content/content-helper.sh | 22 +++++++++++++++++++---
 t/t0012-content-sqlite.t    | 14 ++++++++++++++
 t/t0018-content-files.t     | 16 +++++++++++++++-
 t/t0024-content-s3.t        | 16 +++++++++++++++-
 4 files changed, 63 insertions(+), 5 deletions(-)

diff --git a/t/content/content-helper.sh b/t/content/content-helper.sh
index 2ea80db409d7..761619d44a69 100755
--- a/t/content/content-helper.sh
+++ b/t/content/content-helper.sh
@@ -3,12 +3,28 @@
 
 # content module test helper functions
 
+# Get RPC message contents for checkpoint put
+# Usage: checkpoint_put_msg key rootref
+checkpoint_put_msg() {
+	o="{key:\"$1\",value:{version:1,rootref:\"$2\",timestamp:2.2}}"
+	echo ${o}
+}
+
 # Usage: checkpoint_put key rootref
 checkpoint_put() {
-	o="{key:\"$1\",value:{version:1,rootref:\"$2\",timestamp:2.2}}"
-	jq -j -c -n  ${o} | $RPC content.checkpoint-put
+	o=$(checkpoint_put_msg $1 $2)
+	jq -j -c -n ${o} | $RPC content.checkpoint-put
 }
+
+# Get RPC message contents for checkpoint get
+# Usage: checkpoint_get_msg key
+checkpoint_get_msg() {
+	o="{key:\"$1\"}"
+	echo ${o}
+}
+
 # Usage: checkpoint_get key >value
 checkpoint_get() {
-	jq -j -c -n  "{key:\"$1\"}" | $RPC content.checkpoint-get
+	o=$(checkpoint_get_msg $1)
+	jq -j -c -n ${o} | $RPC content.checkpoint-get
 }
diff --git a/t/t0012-content-sqlite.t b/t/t0012-content-sqlite.t
index 1105dc130f07..3099f1c05147 100755
--- a/t/t0012-content-sqlite.t
+++ b/t/t0012-content-sqlite.t
@@ -133,6 +133,20 @@ test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref bar' '
 	test_cmp rootref.exp rootref.out
 '
 
+test_expect_success HAVE_JQ 'checkpoint-put on rank 1 forwards to rank 0' '
+       o=$(checkpoint_put_msg rankone rankref) &&
+       jq -j -c -n ${o} | flux exec -r 1 ${RPC} content.checkpoint-put
+'
+
+test_expect_success HAVE_JQ 'checkpoint-get on rank 1 forwards to rank 0' '
+       echo rankref >rankref.exp &&
+       o=$(checkpoint_get_msg rankone) &&
+       jq -j -c -n ${o} \
+	   | flux exec -r 1 ${RPC} content.checkpoint-get \
+	   | jq -r .value | jq -r .rootref > rankref.out &&
+       test_cmp rankref.exp rankref.out
+'
+
 # use grep instead of compare, incase of floating point rounding
 test_expect_success HAVE_JQ 'checkpoint-get foo returned correct timestamp' '
         checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out &&
diff --git a/t/t0018-content-files.t b/t/t0018-content-files.t
index 67430f7a9c67..cbb7dead94fb 100755
--- a/t/t0018-content-files.t
+++ b/t/t0018-content-files.t
@@ -6,7 +6,7 @@ test_description='Test content-files backing store service'
 
 . `dirname $0`/sharness.sh
 
-test_under_flux 1 minimal -o,-Sstatedir=$(pwd)
+test_under_flux 2 minimal -o,-Sstatedir=$(pwd)
 
 BLOBREF=${FLUX_BUILD_DIR}/t/kvs/blobref
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
@@ -168,6 +168,20 @@ test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref bar' '
         test_cmp rootref.exp rootref.out
 '
 
+test_expect_success HAVE_JQ 'checkpoint-put on rank 1 forwards to rank 0' '
+       o=$(checkpoint_put_msg rankone rankref) &&
+       jq -j -c -n ${o} | flux exec -r 1 ${RPC} content.checkpoint-put
+'
+
+test_expect_success HAVE_JQ 'checkpoint-get on rank 1 forwards to rank 0' '
+       echo rankref >rankref.exp &&
+       o=$(checkpoint_get_msg rankone) &&
+       jq -j -c -n ${o} \
+	   | flux exec -r 1 ${RPC} content.checkpoint-get \
+	   | jq -r .value | jq -r .rootref > rankref.out &&
+       test_cmp rankref.exp rankref.out
+'
+
 # use grep instead of compare, incase of floating point rounding
 test_expect_success HAVE_JQ 'checkpoint-get foo returned correct timestamp' '
         checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out &&
diff --git a/t/t0024-content-s3.t b/t/t0024-content-s3.t
index 9c8799061754..4047eb90e8aa 100755
--- a/t/t0024-content-s3.t
+++ b/t/t0024-content-s3.t
@@ -13,7 +13,7 @@ fi
 
 export FLUX_CONF_DIR=$(pwd)
 
-test_under_flux 1 minimal
+test_under_flux 2 minimal
 
 BLOBREF=${FLUX_BUILD_DIR}/t/kvs/blobref
 RPC=${FLUX_BUILD_DIR}/t/request/rpc
@@ -127,6 +127,20 @@ test_expect_success HAVE_JQ 'checkpoint-get foo returned rootref bar' '
         test_cmp rootref.exp rootref.out
 '
 
+test_expect_success HAVE_JQ 'checkpoint-put on rank 1 forwards to rank 0' '
+       o=$(checkpoint_put_msg rankone rankref) &&
+       jq -j -c -n ${o} | flux exec -r 1 ${RPC} content.checkpoint-put
+'
+
+test_expect_success HAVE_JQ 'checkpoint-get on rank 1 forwards to rank 0' '
+       echo rankref >rankref.exp &&
+       o=$(checkpoint_get_msg rankone) &&
+       jq -j -c -n ${o} \
+	   | flux exec -r 1 ${RPC} content.checkpoint-get \
+	   | jq -r .value | jq -r .rootref > rankref.out &&
+       test_cmp rankref.exp rankref.out
+'
+
 # use grep instead of compare, incase of floating point rounding
 test_expect_success HAVE_JQ 'checkpoint-get foo returned correct timestamp' '
         checkpoint_get foo | jq -r .value | jq -r .timestamp >timestamp.out &&