Merge pull request #1262 from retroflexer/bz1770273-crictl-error-in-s…

…top-all-containers

Bug 1770273: Verify all containers are dead in stop_all_containers() function

templates/master/00-master/_base/files/usr-local-bin-openshift-recovery-tools-sh.yaml

@@ -337,8 +337,20 @@ contents:
     }
 
     stop_all_containers() {
-      echo "Stopping all containers.."
-      crictl ps -q | xargs -r crictl stop
+      conids=$(crictl ps -q)
+      iterations=0
+      while [ ! -z "$conids" ]; do
+          let iterations=$iterations+1
+          if [ $iterations -ge 60 ]; then
+              echo "Failed to stop all containers after 60 iterations. Exiting!"
+              exit 1
+          fi
+          crictl stop $conids || true
+          echo "Waiting for all containers to stop... ($iterations/60)"
+          sleep 5
+          conids=$(crictl ps -q)
+      done
+      echo "All containers are stopped."
     }
 
     # validate_environment performs the same actions as the discovery container in etcd-member init