From 1d77feb0a008ae23dbe02be3a2f232ae59bb5b48 Mon Sep 17 00:00:00 2001 From: Jeremy Eder Date: Tue, 9 Dec 2025 08:04:12 -0500 Subject: [PATCH] Fix frontend health check timing issue in smoke tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The frontend health check was failing because the test ran immediately after pods reached Ready status, but NodePort routing may not be fully established yet. This is a race condition between pod readiness and service endpoint propagation. Changes: - Added kubectl wait for pod readiness before health checks - Implemented retry logic (5 attempts with 2s delay) for both backend and frontend - Fixed GitHub Actions to properly fail on deployment timeouts This ensures smoke tests wait for actual service availability, not just pod readiness. Fixes: https://github.com/ambient-code/platform/actions/runs/20064017469/job/57547783304 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Sonnet 4.5 --- .github/workflows/test-local-dev.yml | 3 +++ Makefile | 24 ++++++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-local-dev.yml b/.github/workflows/test-local-dev.yml index 0370ff28..d03e583a 100644 --- a/.github/workflows/test-local-dev.yml +++ b/.github/workflows/test-local-dev.yml @@ -44,16 +44,19 @@ jobs: echo "⚠️ Backend deployment timeout - showing status" kubectl get pods -n ambient-code -o wide kubectl describe deployment backend-api -n ambient-code | tail -50 + exit 1 } kubectl wait --for=condition=available --timeout=180s deployment/frontend -n ambient-code || { echo "⚠️ Frontend deployment timeout - showing status" kubectl get pods -n ambient-code -o wide kubectl describe deployment frontend -n ambient-code | tail -50 + exit 1 } kubectl wait --for=condition=available --timeout=180s deployment/agentic-operator -n ambient-code || { echo "⚠️ Operator deployment timeout - showing status" kubectl get pods -n ambient-code -o wide kubectl describe deployment agentic-operator -n ambient-code | tail -50 + exit 1 } - name: Run Makefile smoke tests diff --git a/Makefile b/Makefile index b7c5578e..85707331 100644 --- a/Makefile +++ b/Makefile @@ -355,12 +355,28 @@ local-test-quick: check-kubectl check-minikube ## Quick smoke test of local envi @minikube status >/dev/null 2>&1 && echo "$(COLOR_GREEN)✓$(COLOR_RESET) Minikube running" || (echo "$(COLOR_RED)✗$(COLOR_RESET) Minikube not running" && exit 1) @echo "$(COLOR_BLUE)▶$(COLOR_RESET) Testing namespace..." @kubectl get namespace $(NAMESPACE) >/dev/null 2>&1 && echo "$(COLOR_GREEN)✓$(COLOR_RESET) Namespace exists" || (echo "$(COLOR_RED)✗$(COLOR_RESET) Namespace missing" && exit 1) - @echo "$(COLOR_BLUE)▶$(COLOR_RESET) Testing pods..." - @kubectl get pods -n $(NAMESPACE) 2>/dev/null | grep -q "Running" && echo "$(COLOR_GREEN)✓$(COLOR_RESET) Pods running" || (echo "$(COLOR_RED)✗$(COLOR_RESET) No pods running" && exit 1) + @echo "$(COLOR_BLUE)▶$(COLOR_RESET) Waiting for pods to be ready..." + @kubectl wait --for=condition=ready pod -l app=backend -n $(NAMESPACE) --timeout=60s >/dev/null 2>&1 && \ + kubectl wait --for=condition=ready pod -l app=frontend -n $(NAMESPACE) --timeout=60s >/dev/null 2>&1 && \ + echo "$(COLOR_GREEN)✓$(COLOR_RESET) Pods ready" || (echo "$(COLOR_RED)✗$(COLOR_RESET) Pods not ready" && exit 1) @echo "$(COLOR_BLUE)▶$(COLOR_RESET) Testing backend health..." - @curl -sf http://$$(minikube ip):30080/health >/dev/null 2>&1 && echo "$(COLOR_GREEN)✓$(COLOR_RESET) Backend healthy" || (echo "$(COLOR_RED)✗$(COLOR_RESET) Backend not responding" && exit 1) + @for i in 1 2 3 4 5; do \ + curl -sf http://$$(minikube ip):30080/health >/dev/null 2>&1 && { echo "$(COLOR_GREEN)✓$(COLOR_RESET) Backend healthy"; break; } || { \ + if [ $$i -eq 5 ]; then \ + echo "$(COLOR_RED)✗$(COLOR_RESET) Backend not responding after 5 attempts"; exit 1; \ + fi; \ + sleep 2; \ + }; \ + done @echo "$(COLOR_BLUE)▶$(COLOR_RESET) Testing frontend..." - @curl -sf http://$$(minikube ip):30030 >/dev/null 2>&1 && echo "$(COLOR_GREEN)✓$(COLOR_RESET) Frontend accessible" || (echo "$(COLOR_RED)✗$(COLOR_RESET) Frontend not responding" && exit 1) + @for i in 1 2 3 4 5; do \ + curl -sf http://$$(minikube ip):30030 >/dev/null 2>&1 && { echo "$(COLOR_GREEN)✓$(COLOR_RESET) Frontend accessible"; break; } || { \ + if [ $$i -eq 5 ]; then \ + echo "$(COLOR_RED)✗$(COLOR_RESET) Frontend not responding after 5 attempts"; exit 1; \ + fi; \ + sleep 2; \ + }; \ + done @echo "" @echo "$(COLOR_GREEN)✓ Quick smoke test passed!$(COLOR_RESET)"