Make decommission test more reliable (#2018)

testing/common.sh

@@ -148,7 +148,7 @@ function wait_for_n_tenant_pods() {
     try kubectl wait --namespace "$NAMESPACE" \
         --for=condition=ready pod \
         --selector v1.min.io/tenant="$TENANT_NAME" \
-        --timeout=300s
+        --timeout=600s
 }
 
 # copies the script to the pod.

testing/decommission-test.sh

@@ -59,12 +59,26 @@ function main() {
 
     echo -e "\n\n"
     echo "decommission-test.sh: main(): Verify data after decommission:"
-    execute_pod_script verify-data.sh "${DEBUG_POD_NAME}"
-    RESULT=$(read_script_result default "${DEBUG_POD_NAME}" verify-data.log)
+    RESULT="script failed" # The initial state is failure.
+    counter=0
+    while [ $counter -lt 60 ]; do
+        echo "Decommission Check Attempt: $counter"
+        ((counter++))
+        execute_pod_script verify-data.sh "${DEBUG_POD_NAME}"
+        RESULT=$(read_script_result default "${DEBUG_POD_NAME}" verify-data.log)
+        if [ "$RESULT" == "script passed" ]; then
+            break
+        fi
+        sleep 1
+    done
     teardown distributed
     if [ "$RESULT" != "script passed" ]; then
+        echo "Decommission Test Failed"
         exit 1
     fi
+    echo "Decomission Test Passed"
+    exit 0
+
 }
 
 main "$@"

testing/verify-data.sh

@@ -7,22 +7,24 @@ echo "Verify Data in remaining pool(s) after decommission test"
 echo "Get data and verify files are still present as they were uploaded"
 FILE_TO_SAVE_MC_OUTPUT=tmp.log
 rm -rf $FILE_TO_SAVE_MC_OUTPUT
-mc ls myminio/bucket >$FILE_TO_SAVE_MC_OUTPUT 2>&1
+mc ls myminio/bucket > $FILE_TO_SAVE_MC_OUTPUT 2>&1
 RESULT=$(grep -c ERROR $FILE_TO_SAVE_MC_OUTPUT | awk -F' ' '{print $1}')
 while [ "$RESULT" == "1" ]; do
 	echo "There was an error in mc ls, retrying"
-	mc ls myminio/bucket >$FILE_TO_SAVE_MC_OUTPUT 2>&1
+	mc ls myminio/bucket > $FILE_TO_SAVE_MC_OUTPUT 2>&1
 	RESULT=$(grep -c ERROR $FILE_TO_SAVE_MC_OUTPUT | awk -F' ' '{print $1}')
 	echo "${RESULT}"
 done
 echo "mc ls was successful"
+mc ls myminio/bucket # To see what is listed
 
 for i in {1..10}; do
 	RESULT=$(grep -c file"$i" "${FILE_TO_SAVE_MC_OUTPUT}")
 	if [ "${RESULT}" -eq 0 ]; then
 		echo "fail, there is a missing file: file${i}"
 		echo "exiting with 1 as there is missing file..."
+		echo "script failed" > verify-data.log
 		exit 1
 	fi
 done
-echo "script passed" >verify-data.log
+echo "script passed" > verify-data.log