chore: adding unit tests on infra package

.github/workflows/e2e.yaml

@@ -5,6 +5,7 @@ jobs:
   run:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         tests:
         - TestBuildBundle
@@ -45,6 +46,11 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: "1.21.0"
+    - name: Free up runner disk space
+      run: |
+        sudo rm -rf /usr/local/lib/android
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        sudo rm -rf /opt/hostedtoolcache/Python
     - name: E2E
       run: |
         make e2e-test TEST_NAME=${{ matrix.tests }}

Makefile

@@ -154,7 +154,7 @@ e2e-test: helmvm-linux-amd64
 	mkdir -p output/tmp
 	rm -rf output/tmp/id_rsa*
 	ssh-keygen -t rsa -N "" -C "Integration Test Key" -f output/tmp/id_rsa
-	go test -timeout 30m -v ./e2e -run $(TEST_NAME)$
+	go test -timeout 45m -v ./e2e -run $(TEST_NAME)$
 
 .PHONY: create-e2e-workflows
 create-e2e-workflows:

e2e/embed_test.go

@@ -27,11 +27,6 @@ func TestEmbedAndInstall(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install embedded ssh in node 0: %v", err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestEmbedAddonsOnly(t *testing.T) {

e2e/install_test.go

@@ -87,11 +87,6 @@ func TestSingleNodeInstallation(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestMultiNodeInstallation(t *testing.T) {
@@ -145,11 +140,6 @@ func TestSingleNodeInstallationRockyLinux8(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestSingleNodeInstallationDebian12(t *testing.T) {
@@ -174,10 +164,6 @@ func TestSingleNodeInstallationDebian12(t *testing.T) {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
 	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestSingleNodeInstallationCentos8Stream(t *testing.T) {
@@ -205,11 +191,6 @@ func TestSingleNodeInstallationCentos8Stream(t *testing.T) {
 	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
 		t.Fatalf("fail to install helmvm on node %s: %v", tc.Nodes[0], err)
 	}
-	t.Log("creating deployment mounting pvc")
-	line = []string{"deploy-with-pvc.sh"}
-	if _, _, err := RunCommandOnNode(t, tc, 0, line); err != nil {
-		t.Fatalf("fail to create deployment with pvc: %v", err)
-	}
 }
 
 func TestMultiNodeInteractiveInstallation(t *testing.T) {

e2e/scripts/addons-only.sh

@@ -53,16 +53,16 @@ embed_helm_chart() {
 }
 
 wait_for_memcached_pods() {
-    ready=$(kubectl get pods -n helmvm | grep -v NotReady | grep Ready | grep -c memcached || true)
+    ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
     counter=0
     while [ -z "$ready" ] || [ "$ready" -lt "1" ]; do
         if [ "$counter" -gt 36 ]; then
             return 1
         fi
         sleep 5
         counter=$((counter+1))
-        echo "Waiting for memcached pods to be ready"
-        ready=$(kubectl get pods -n helmvm | grep  Running | grep -c memcached || true)
+        echo "Waiting for memcached pods"
+        ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
         kubectl get pods -n helmvm 2>&1 || true
         echo "$ready"
     done
@@ -97,7 +97,7 @@ main() {
     fi
     echo "waiting for memcached " >> /tmp/log
     if ! wait_for_memcached_pods; then
-        echo "Memcached pods not reporting healthy"
+        echo "Memcached pods not appearing"
         exit 1
     fi
 }

e2e/scripts/embed-and-install.sh

@@ -54,22 +54,50 @@ embed_helm_chart() {
 }
 
 wait_for_memcached_pods() {
-    ready=$(kubectl get pods -n helmvm | grep -v NotReady | grep Ready | grep -c memcached || true)
+    ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
     counter=0
     while [ "$ready" -lt "1" ]; do
         if [ "$counter" -gt 36 ]; then
             return 1
         fi
         sleep 5
         counter=$((counter+1))
-        echo "Waiting for memcached pods to be ready"
-        ready=$(kubectl get pods -n helmvm | grep  Running | grep -c memcached || true)
+        echo "Waiting for memcached pods"
+        ready=$(kubectl get pods -n helmvm | grep -c memcached || true)
         kubectl get pods -n helmvm 2>&1 || true
         echo "$ready"
     done
     return 0
 }
 
+wait_for_pods_running() {
+    local timeout="$1"
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            kubectl get pods -A -o yaml || true
+            kubectl describe nodes || true
+            echo "Timed out waiting for all pods to be running."
+            return 1
+        fi
+        local non_running_pods
+        non_running_pods=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { print $0 }' | wc -l || echo 1)
+        if [ "$non_running_pods" -ne 0 ]; then
+            echo "Not all pods are running. Waiting."
+            kubectl get pods,nodes -A || true
+            sleep 5
+            continue
+        fi
+        echo "All pods are running."
+        return 0
+    done
+}
+
 main() {
     if ! install_helm ; then
         echo "Failed to install helm"
@@ -96,9 +124,13 @@ main() {
         echo "Nodes not reporting healthy"
         exit 1
     fi
+    if ! wait_for_pods_running 900; then
+        echo "Pods not running"
+        exit 1
+    fi
     echo "waiting for memcached " >> /tmp/log
     if ! wait_for_memcached_pods; then
-        echo "Memcached pods not reporting healthy"
+        echo "Memcached pods not present"
         exit 1
     fi
 }

e2e/scripts/single-node-install.sh

@@ -17,6 +17,34 @@ wait_for_healthy_node() {
     return 0
 }
 
+wait_for_pods_running() {
+    local timeout="$1"
+    local start_time
+    local current_time
+    local elapsed_time
+    start_time=$(date +%s)
+    while true; do
+        current_time=$(date +%s)
+        elapsed_time=$((current_time - start_time))
+        if [ "$elapsed_time" -ge "$timeout" ]; then
+            kubectl get pods -A -o yaml || true
+            kubectl describe nodes || true
+            echo "Timed out waiting for all pods to be running."
+            return 1
+        fi
+        local non_running_pods
+        non_running_pods=$(kubectl get pods --all-namespaces --no-headers 2>/dev/null | awk '$4 != "Running" && $4 != "Completed" { print $0 }' | wc -l || echo 1)
+        if [ "$non_running_pods" -ne 0 ]; then
+            echo "Not all pods are running. Waiting."
+            kubectl get pods,nodes -A || true
+            sleep 5
+            continue
+        fi
+        echo "All pods are running."
+        return 0
+    done
+}
+
 main() {
     if ! helmvm install --no-prompt 2>&1 | tee /tmp/log ; then
         cat /etc/os-release
@@ -31,6 +59,10 @@ main() {
         echo "Failed to install helmvm"
         exit 1
     fi
+    if ! wait_for_pods_running 900; then
+        echo "Failed to install helmvm"
+        exit 1
+    fi
 }
 
 export KUBECONFIG=/root/.helmvm/etc/kubeconfig

e2e/utils.go

@@ -38,7 +38,7 @@ func RunCommandOnNode(t *testing.T, cl *cluster.Output, node int, line []string)
 		Stdout: stdout,
 		Stderr: stderr,
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Minute)
 	defer cancel()
 	if err := cluster.Run(ctx, t, cmd); err != nil {
 		t.Logf("stdout:\n%s", stdout.String())