CI: Wait and Retry

test/e2e/pipeline/infra/cluster-api/cni.sh

@@ -7,6 +7,9 @@ WORKDIR=$(dirname "$FILEPATH")
 # shellcheck source=./pre-requirements.sh
 source "$WORKDIR/pre-requirements.sh"
 
+# shellcheck source=../../utils.sh
+source "$WORKDIR/../../utils.sh"
+
 DOCKER_PROXY="${DOCKER_PROXY:-docker.io}"
 
 function install_calico() {
@@ -56,11 +59,17 @@ EOF
 
 function wait_calico() {
     local kubeconfig=$1
-    sleep 5
-    "${KUBECTL}" wait --for condition=Ready=true -n calico-system pod --all --kubeconfig "$kubeconfig" --timeout=-1s
-    sleep 10
+    if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n calico-system pod --all --kubeconfig $kubeconfig --timeout=-1s"
+    then
+      echo "Failed to wait for calico pods to be ready"
+      exit 1
+    fi
     # set felix to use different port for VXLAN
-    "${KUBECTL}" patch felixconfiguration default --type='merge' -p '{"spec":{"vxlanPort": 6789}}' --kubeconfig "$kubeconfig"
+    if ! waitandretry 5s 12 "${KUBECTL} patch felixconfiguration default --type='merge' -p '{\"spec\":{\"vxlanPort\": 6789}}' --kubeconfig $kubeconfig";
+    then
+      echo "Failed to patch felixconfiguration"
+      exit 1
+    fi
 }
 
 function install_cilium() {
@@ -112,6 +121,9 @@ function install_flannel() {
 
 function wait_flannel() {
     local kubeconfig=$1
-    sleep 15
-    "${KUBECTL}" wait --for condition=Ready=true -n kube-flannel pod --all --timeout=-1s --kubeconfig "$kubeconfig"
+    if ! waitandretry 5s 12 "${KUBECTL} wait --for condition=Ready=true -n kube-flannel pod --all --timeout=-1s --kubeconfig $kubeconfig";
+    then
+      echo "Failed to wait for flannel pods to be ready"
+      exit 1
+    fi
 }

test/e2e/pipeline/utils.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+#shellcheck disable=SC1091
+
+# Define the retry function
+waitandretry() {
+  local waittime="$1"
+  local retries="$2"
+  local command="$3"
+  local options="$-" # Get the current "set" options
+
+  sleep "${waittime}"
+
+  echo "Running command: ${command} (retries left: ${retries})"
+
+  # Disable set -e
+  if [[ $options == *e* ]]; then
+    set +e
+  fi
+
+  # Run the command, and save the exit code
+  $command
+  local exit_code=$?
+
+  # restore initial options
+  if [[ $options == *e* ]]; then
+    set -e
+  fi
+
+  # If the exit code is non-zero (i.e. command failed), and we have not
+  # reached the maximum number of retries, run the command again
+  if [[ $exit_code -ne 0 && $retries -gt 0 ]]; then
+    waitandretry "$waittime" $((retries - 1)) "$command"
+  else
+    # Return the exit code from the command
+    return $exit_code
+  fi
+}