Merge #3548

3548: Add retries to all cURL requests that connect to NCTL node r=rafal-ch a=rafal-ch This PR updates the NCTL scripts to allow retries when trying to read data from the node by using cURL. There are cases when the node is not able to process requests (usually immediately after restart) and this could lead to a test being stuck. Co-authored-by: Rafał Chabowski <rafal@casperlabs.io>
casper-network · Jan 7, 2023 · 01c2af5 · 01c2af5
2 parents 993ba53 + 7f6406f
commit 01c2af5
Show file tree

Hide file tree

Showing 8 changed files with 22 additions and 13 deletions.
diff --git a/utils/nctl/sh/utils/constants.sh b/utils/nctl/sh/utils/constants.sh
@@ -78,3 +78,10 @@ export NCTL_PROCESS_GROUP_2=validators-2
 
 # Name of process group: non-genesis validators.
 export NCTL_PROCESS_GROUP_3=validators-3
+
+# cURL arguments which are used when talking to the NCTL nodes.
+# We need to allow retires and limit the default timeouts because not all
+# testing scenarios guarantee that nodes are responsive immediately, which may
+# lead to the test being stuck.
+# In addition, we don't want cURL to put anything on the standard output.
+export NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES="--max-time 4 --connect-timeout 2 --retry 20 -s"
diff --git a/utils/nctl/sh/utils/queries.sh b/utils/nctl/sh/utils/queries.sh
@@ -26,9 +26,11 @@ function get_chain_era()
 {
     local NODE_ID=${1:-$(get_node_for_dispatch)}
     local ERA
+    local STATUS
 
     if [ "$(get_node_is_up "$NODE_ID")" = true ]; then
-        ERA=$(curl -s "$(get_node_address_rest $NODE_ID)/status" | jq '.last_added_block_info.era_id')
+        STATUS=$(curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES "$(get_node_address_rest $NODE_ID)/status")
+        ERA=$(echo $STATUS | jq '.last_added_block_info.era_id')
         if [ "$ERA" == "null" ]; then
             echo 0
         else
@@ -50,7 +52,7 @@ function get_chain_height()
     local NODE_ID=${1:-$(get_node_for_dispatch)}
 
     if [ "$(get_node_is_up "$NODE_ID")" = true ]; then
-        curl -s "$(get_node_address_rest $NODE_ID)/status" | jq '.last_added_block_info.height'
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES "$(get_node_address_rest $NODE_ID)/status" | jq '.last_added_block_info.height'
     else
         echo "N/A"
     fi
@@ -75,7 +77,7 @@ function get_chain_latest_block_hash()
 {
     local NODE_ID=${1:-$(get_node_for_dispatch)}
 
-    curl -s "$(get_node_address_rest $NODE_ID)/status" \
+    curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES "$(get_node_address_rest $NODE_ID)/status" \
     | jq '.last_added_block_info.hash' \
     | tr '[:upper:]' '[:lower:]' \
     | sed -e 's/^"//' -e 's/"$//'
@@ -133,7 +135,7 @@ function get_node_status()
 
     NODE_ADDRESS_CURL=$(get_node_address_rpc_for_curl "$NODE_ID")
     NODE_API_RESPONSE=$(
-        curl -s --header 'Content-Type: application/json' \
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
             --request POST "$NODE_ADDRESS_CURL" \
             --data-raw '{
                 "id": 1,
@@ -229,6 +231,6 @@ function get_node_connected_peer_count()
 {
     local NODE_ID=${1}
 
-    echo $(curl -s "$(get_node_address_rest $NODE_ID)/status" | jq '.peers' | jq length)
+    echo $(curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES "$(get_node_address_rest $NODE_ID)/status" | jq '.peers' | jq length)
 }
 
diff --git a/utils/nctl/sh/views/view_node_metrics.sh b/utils/nctl/sh/views/view_node_metrics.sh
@@ -44,9 +44,9 @@ function do_render()
     ENDPOINT="$(get_node_address_rest "$NODE_ID")"/metrics
 
     if [ "$METRICS" = "all" ]; then
-        curl -s --location --request GET "$ENDPOINT"  
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --location --request GET "$ENDPOINT"  
     else
-        echo "node #$NODE_ID :: $(curl -s --location --request GET "$ENDPOINT" | grep "$METRICS" | tail -n 1)"
+        echo "node #$NODE_ID :: $(curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --location --request GET "$ENDPOINT" | grep "$METRICS" | tail -n 1)"
     fi
 }
 

diff --git a/utils/nctl/sh/views/view_node_peer_count.sh b/utils/nctl/sh/views/view_node_peer_count.sh
@@ -34,7 +34,7 @@ function do_render()
 
     NODE_ADDRESS_CURL=$(get_node_address_rpc_for_curl "$NODE_ID")
     NODE_PEER_COUNT=$(
-        curl -s --header 'Content-Type: application/json' \
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
             --request POST "$NODE_ADDRESS_CURL" \
             --data-raw '{
                 "id": 1,

diff --git a/utils/nctl/sh/views/view_node_peers.sh b/utils/nctl/sh/views/view_node_peers.sh
@@ -38,7 +38,7 @@ function do_render()
 
     NODE_ADDRESS_CURL=$(get_node_address_rpc_for_curl "$NODE_ID")
     NODE_API_RESPONSE=$(
-        curl -s --header 'Content-Type: application/json' \
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
             --request POST "$NODE_ADDRESS_CURL" \
             --data-raw '{
                 "id": 1,

diff --git a/utils/nctl/sh/views/view_node_rpc_endpoint.sh b/utils/nctl/sh/views/view_node_rpc_endpoint.sh
@@ -10,15 +10,15 @@ function main()
     local ENDPOINT=${1}
 
     if [ "$ENDPOINT" = "all" ]; then
-        curl -s --header 'Content-Type: application/json' \
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
             --request POST "$(get_node_address_rpc_for_curl)" \
             --data-raw '{
                 "id": 1,
                 "jsonrpc": "2.0",
                 "method": "rpc.discover"
             }' | jq '.result.schema.methods[].name'
     else
-        curl -s --header 'Content-Type: application/json' \
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
             --request POST "$(get_node_address_rpc_for_curl)" \
             --data-raw '{
                 "id": 1,

diff --git a/utils/nctl/sh/views/view_node_rpc_schema.sh b/utils/nctl/sh/views/view_node_rpc_schema.sh
@@ -7,7 +7,7 @@ source "$NCTL"/sh/utils/main.sh
 #######################################
 function main()
 {
-    curl -s --header 'Content-Type: application/json' \
+    curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
         --request POST "$(get_node_address_rpc_for_curl)" \
         --data-raw '{
             "id": 1,

diff --git a/utils/nctl/sh/views/view_node_status.sh b/utils/nctl/sh/views/view_node_status.sh
@@ -41,7 +41,7 @@ function do_render()
 
     NODE_ADDRESS_CURL=$(get_node_address_rpc_for_curl "$NODE_ID")
     NODE_API_RESPONSE=$(
-        curl -s --header 'Content-Type: application/json' \
+        curl $NCTL_CURL_ARGS_FOR_NODE_RELATED_QUERIES --header 'Content-Type: application/json' \
             --request POST "$NODE_ADDRESS_CURL" \
             --data-raw '{
                 "id": 1,