wait for NFS PVs to be deleted before continuing

Intended to help with flakes, but didn't make a difference.
Probably still worth doing.

Signed-off-by: Maya Rashish <mrashish@redhat.com>

tests/framework/nfs-utils.go

@@ -29,7 +29,11 @@ func createNFSPVs(client *kubernetes.Clientset, cdiNs string) error {
 
 func deleteNFSPVs(client *kubernetes.Clientset, cdiNs string) error {
 	for i := 1; i <= pvCount; i++ {
-		if err := utils.DeletePV(client, nfsPVDef(strconv.Itoa(i), utils.NfsService.Spec.ClusterIP)); err != nil {
+		pv := nfsPVDef(strconv.Itoa(i), utils.NfsService.Spec.ClusterIP)
+		if err := utils.DeletePV(client, pv); err != nil {
+			return err
+		}
+		if err := utils.WaitTimeoutForPVDeleted(client, pv, timeout); err != nil {
 			return err
 		}
 	}

tests/utils/pv.go

@@ -83,6 +83,17 @@ func WaitTimeoutForPVStatus(clientSet *kubernetes.Clientset, pvName string, stat
 	return wait.PollImmediate(2*time.Second, timeout, pvPhase(clientSet, pvName, status))
 }
 
+// WaitTimeoutForPVDeleted waits until the given pv no longer exists
+func WaitTimeoutForPVDeleted(clientSet *kubernetes.Clientset, pv *k8sv1.PersistentVolume, timeout time.Duration) error {
+	return wait.PollImmediate(pvPollInterval, timeout, func() (bool, error) {
+		err := clientSet.CoreV1().PersistentVolumes().Delete(context.TODO(), pv.GetName(), metav1.DeleteOptions{})
+		if apierrs.IsNotFound(err) {
+			return true, nil
+		}
+		return false, err
+	})
+}
+
 func pvPhase(clientSet *kubernetes.Clientset, pvName string, status k8sv1.PersistentVolumePhase) wait.ConditionFunc {
 	return func() (bool, error) {
 		pv, err := clientSet.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})