Add logging format and tests for k8s/deployment module

k8s/apply_templates

@@ -1,12 +1,25 @@
 #!/bin/bash
 
-echo "TEMPLATE DIR: $OUTPUT_DIR, ACTION: $ACTION, DRY_RUN: $DRY_RUN"
+echo "📝 Applying templates..."
+echo "📋 Directory: $OUTPUT_DIR"
+echo "📋 Action: $ACTION"
+echo "📋 Dry run: $DRY_RUN"
+echo ""
 
 APPLIED_FILES=()
 
 # Find all .yaml files that were not yet applied / deleted
 while IFS= read -r TEMPLATE_FILE; do
-   echo "kubectl $ACTION $TEMPLATE_FILE"
+   FILENAME="$(basename "$TEMPLATE_FILE")"
+   BASE_DIR="$(dirname "$TEMPLATE_FILE")"
+
+   # Check if file is empty or contains only whitespace
+   if [[ ! -s "$TEMPLATE_FILE" ]] || [[ -z "$(tr -d '[:space:]' < "$TEMPLATE_FILE")" ]]; then
+     echo "📋 Skipping empty template: $FILENAME"
+     continue
+   fi
+
+   echo "📝 kubectl $ACTION $FILENAME"
 
    if [[ "$DRY_RUN" == "false" ]]; then
      IGNORE_NOT_FOUND=""
@@ -15,11 +28,11 @@ while IFS= read -r TEMPLATE_FILE; do
        IGNORE_NOT_FOUND="--ignore-not-found=true"
      fi
 
-     kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND
+     if ! kubectl "$ACTION" -f "$TEMPLATE_FILE" $IGNORE_NOT_FOUND; then
+       echo "   ❌ Failed to apply"
+     fi
    fi
 
-   BASE_DIR="$(dirname "$TEMPLATE_FILE")"
-   FILENAME="$(basename "$TEMPLATE_FILE")"
    DEST_DIR="${BASE_DIR}/$ACTION"
 
    mkdir -p "$DEST_DIR"
@@ -31,6 +44,8 @@ while IFS= read -r TEMPLATE_FILE; do
 done < <(find "$OUTPUT_DIR" \( -path "*/apply" -o -path "*/delete" \) -prune -o -type f -name "*.yaml" -print)
 
 if [[ "$DRY_RUN" == "true" ]]; then
+  echo ""
+  echo "📋 Dry run mode - no changes were made"
   exit 1
 fi
 

k8s/deployment/build_context

@@ -20,7 +20,7 @@ SWITCH_TRAFFIC=$(echo "$CONTEXT" | jq -r ".deployment.strategy_data.desired_swit
 MIN_REPLICAS=$(echo "scale=10; $REPLICAS / 10" | bc)
 MIN_REPLICAS=$(echo "$MIN_REPLICAS" | awk '{printf "%d", ($1 == int($1) ? $1 : int($1)+1)}')
 
-DEPLOYMENT_STATUS=$(echo $CONTEXT | jq -r ".deployment.status")
+DEPLOYMENT_STATUS=$(echo "$CONTEXT" | jq -r ".deployment.status")
 
 validate_status() {
   local action="$1"
@@ -44,12 +44,12 @@ validate_status() {
       expected_status="deleting, rolling_back or cancelling"
       ;;
     *)
-      echo "🔄 Running action '$action', any deployment status is accepted"
+      echo "📝 Running action '$action', any deployment status is accepted"
       return 0
       ;;
   esac
 
-  echo "🔄 Running action '$action' (current status: '$status', expected: $expected_status)"
+  echo "📝 Running action '$action' (current status: '$status', expected: $expected_status)"
 
   case "$action" in
     start-initial|start-blue-green)
@@ -72,6 +72,14 @@ validate_status() {
 
 if ! validate_status "$SERVICE_ACTION" "$DEPLOYMENT_STATUS"; then
   echo "❌ Invalid deployment status '$DEPLOYMENT_STATUS' for action '$SERVICE_ACTION'" >&2
+  echo "💡 Possible causes:" >&2
+  echo "   - Deployment status changed during workflow execution" >&2
+  echo "   - Another action is already running on this deployment" >&2
+  echo "   - Deployment was modified externally" >&2
+  echo "🔧 How to fix:" >&2
+  echo "   - Wait for any in-progress actions to complete" >&2
+  echo "   - Check the deployment status in the nullplatform dashboard" >&2
+  echo "   - Retry the action once the deployment is in the expected state" >&2
   exit 1
 fi
 
@@ -95,7 +103,6 @@ fi
 if [[ -n "$PULL_SECRETS" ]]; then
   IMAGE_PULL_SECRETS=$PULL_SECRETS
 else
-  # Use env var if set, otherwise build from flat properties
   if [ -n "${IMAGE_PULL_SECRETS:-}" ]; then
     IMAGE_PULL_SECRETS=$(echo "$IMAGE_PULL_SECRETS" | jq .)
   else
@@ -191,7 +198,14 @@ if [[ -n "$TRAFFIC_MANAGER_CONFIG_MAP" ]]; then
 
     # Check if the ConfigMap exists
     if ! kubectl get configmap "$TRAFFIC_MANAGER_CONFIG_MAP" -n "$K8S_NAMESPACE" &>/dev/null; then
-        echo "❌ ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' does not exist in namespace '$K8S_NAMESPACE'"
+        echo "❌ ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' does not exist in namespace '$K8S_NAMESPACE'" >&2
+        echo "💡 Possible causes:" >&2
+        echo "   - ConfigMap was not created before deployment" >&2
+        echo "   - ConfigMap name is misspelled in values.yaml" >&2
+        echo "   - ConfigMap was deleted or exists in a different namespace" >&2
+        echo "🔧 How to fix:" >&2
+        echo "   - Create the ConfigMap: kubectl create configmap $TRAFFIC_MANAGER_CONFIG_MAP -n $K8S_NAMESPACE --from-file=nginx.conf --from-file=default.conf" >&2
+        echo "   - Verify the ConfigMap name in your scope configuration" >&2
         exit 1
     fi
     echo "✅ ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' exists"
@@ -204,14 +218,19 @@ if [[ -n "$TRAFFIC_MANAGER_CONFIG_MAP" ]]; then
 
     for key in "${REQUIRED_KEYS[@]}"; do
         if ! echo "$CONFIGMAP_KEYS" | grep -qx "$key"; then
-            echo "❌ ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' is missing required key '$key'"
-            echo "💡 The ConfigMap must contain data entries for: ${REQUIRED_KEYS[*]}"
+            echo "❌ ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' is missing required key '$key'" >&2
+            echo "💡 Possible causes:" >&2
+            echo "   - ConfigMap was created without all required files" >&2
+            echo "   - Key name is different from expected: ${REQUIRED_KEYS[*]}" >&2
+            echo "🔧 How to fix:" >&2
+            echo "   - Update the ConfigMap to include the missing key '$key'" >&2
+            echo "   - Required keys: ${REQUIRED_KEYS[*]}" >&2
             exit 1
         fi
         echo "✅ Found required key '$key' in ConfigMap"
     done
 
-    echo "🎉 ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' validation successful"
+    echo "✨ ConfigMap '$TRAFFIC_MANAGER_CONFIG_MAP' validation successful"
 fi
 
 CONTEXT=$(echo "$CONTEXT" | jq \
@@ -249,3 +268,6 @@ export DEPLOYMENT_ID
 export BLUE_DEPLOYMENT_ID
 
 mkdir -p "$OUTPUT_DIR"
+
+echo "✨ Deployment context built successfully"
+echo "📋 Deployment ID: $DEPLOYMENT_ID | Replicas: green=$GREEN_REPLICAS, blue=$BLUE_REPLICAS"

k8s/deployment/build_deployment

@@ -7,9 +7,11 @@ SERVICE_TEMPLATE_PATH="$OUTPUT_DIR/service-$SCOPE_ID-$DEPLOYMENT_ID.yaml"
 PDB_PATH="$OUTPUT_DIR/pdb-$SCOPE_ID-$DEPLOYMENT_ID.yaml"
 CONTEXT_PATH="$OUTPUT_DIR/context-$SCOPE_ID.json"
 
-echo "$CONTEXT" | jq --arg replicas "$REPLICAS" '. + {replicas: $replicas}' > "$CONTEXT_PATH"
+echo "📝 Building deployment templates..."
+echo "📋 Output directory: $OUTPUT_DIR"
+echo ""
 
-echo "Building Template: $DEPLOYMENT_TEMPLATE to $DEPLOYMENT_PATH"
+echo "$CONTEXT" | jq --arg replicas "$REPLICAS" '. + {replicas: $replicas}' > "$CONTEXT_PATH"
 
 gomplate -c .="$CONTEXT_PATH" \
   --file "$DEPLOYMENT_TEMPLATE" \
@@ -18,11 +20,10 @@ gomplate -c .="$CONTEXT_PATH" \
 TEMPLATE_GENERATION_STATUS=$?
 
 if [[ $TEMPLATE_GENERATION_STATUS -ne 0 ]]; then
-    echo "Error building deployment template"
+    echo "   ❌ Failed to build deployment template"
     exit 1
 fi
-
-echo "Building Template: $SECRET_TEMPLATE to $SECRET_PATH"
+echo "   ✅ Deployment template: $DEPLOYMENT_PATH"
 
 gomplate -c .="$CONTEXT_PATH" \
   --file "$SECRET_TEMPLATE" \
@@ -31,11 +32,10 @@ gomplate -c .="$CONTEXT_PATH" \
 TEMPLATE_GENERATION_STATUS=$?
 
 if [[ $TEMPLATE_GENERATION_STATUS -ne 0 ]]; then
-    echo "Error building secret template"
+    echo "   ❌ Failed to build secret template"
     exit 1
 fi
-
-echo "Building Template: $SCALING_TEMPLATE to $SCALING_PATH"
+echo "   ✅ Secret template: $SECRET_PATH"
 
 gomplate -c .="$CONTEXT_PATH" \
   --file "$SCALING_TEMPLATE" \
@@ -44,11 +44,10 @@ gomplate -c .="$CONTEXT_PATH" \
 TEMPLATE_GENERATION_STATUS=$?
 
 if [[ $TEMPLATE_GENERATION_STATUS -ne 0 ]]; then
-    echo "Error building scaling template"
+    echo "   ❌ Failed to build scaling template"
     exit 1
 fi
-
-echo "Building Template: $SERVICE_TEMPLATE to $SERVICE_TEMPLATE_PATH"
+echo "   ✅ Scaling template: $SCALING_PATH"
 
 gomplate -c .="$CONTEXT_PATH" \
   --file "$SERVICE_TEMPLATE" \
@@ -57,21 +56,25 @@ gomplate -c .="$CONTEXT_PATH" \
 TEMPLATE_GENERATION_STATUS=$?
 
 if [[ $TEMPLATE_GENERATION_STATUS -ne 0 ]]; then
-    echo "Error building service template"
+    echo "   ❌ Failed to build service template"
     exit 1
 fi
+echo "   ✅ Service template: $SERVICE_TEMPLATE_PATH"
 
-echo "Building Template: $PDB_TEMPLATE to $PDB_PATH"
-
+echo "📝 Building PDB template..."
 gomplate -c .="$CONTEXT_PATH" \
   --file "$PDB_TEMPLATE" \
   --out "$PDB_PATH"
 
 TEMPLATE_GENERATION_STATUS=$?
 
 if [[ $TEMPLATE_GENERATION_STATUS -ne 0 ]]; then
-    echo "Error building PDB template"
+    echo "   ❌ Failed to build PDB template"
     exit 1
 fi
+echo "   ✅ PDB template: $PDB_PATH"
 
 rm "$CONTEXT_PATH"
+
+echo ""
+echo "✨ All templates built successfully"

k8s/deployment/delete_cluster_objects

@@ -1,12 +1,28 @@
 #!/bin/bash
 
+echo "🔍 Starting cluster objects cleanup..."
+
 OBJECTS_TO_DELETE="deployment,service,hpa,ingress,pdb,secret,configmap"
 
 # Function to delete all resources for a given deployment_id
 delete_deployment_resources() {
     local DEPLOYMENT_ID_TO_DELETE="$1"
-    kubectl delete "$OBJECTS_TO_DELETE" \
-      -l deployment_id="$DEPLOYMENT_ID_TO_DELETE" -n "$K8S_NAMESPACE" --cascade=foreground --wait=true
+    echo "📝 Deleting resources for deployment_id=$DEPLOYMENT_ID_TO_DELETE..."
+
+    if ! kubectl delete "$OBJECTS_TO_DELETE" \
+      -l deployment_id="$DEPLOYMENT_ID_TO_DELETE" -n "$K8S_NAMESPACE" --cascade=foreground --wait=true; then
+        echo "❌ Failed to delete resources for deployment_id=$DEPLOYMENT_ID_TO_DELETE" >&2
+        echo "💡 Possible causes:" >&2
+        echo "   - Resources may have finalizers preventing deletion" >&2
+        echo "   - Network connectivity issues with Kubernetes API" >&2
+        echo "   - Insufficient permissions to delete resources" >&2
+        echo "🔧 How to fix:" >&2
+        echo "   - Check for stuck finalizers: kubectl get all -l deployment_id=$DEPLOYMENT_ID_TO_DELETE -n $K8S_NAMESPACE -o yaml | grep finalizers" >&2
+        echo "   - Verify kubeconfig and cluster connectivity" >&2
+        echo "   - Check RBAC permissions for the service account" >&2
+        return 1
+    fi
+    echo "✅ Resources deleted for deployment_id=$DEPLOYMENT_ID_TO_DELETE"
 }
 
 CURRENT_ACTIVE=$(echo "$CONTEXT" | jq -r '.scope.current_active_deployment // empty')
@@ -15,15 +31,21 @@ if [ "$DEPLOYMENT" = "blue" ]; then
     # Deleting blue (old) deployment, keeping green (new)
     DEPLOYMENT_TO_CLEAN="$CURRENT_ACTIVE"
     DEPLOYMENT_TO_KEEP="$DEPLOYMENT_ID"
+    echo "📋 Strategy: Deleting blue (old) deployment, keeping green (new)"
 elif [ "$DEPLOYMENT" = "green" ]; then
     # Deleting green (new) deployment, keeping blue (old)
     DEPLOYMENT_TO_CLEAN="$DEPLOYMENT_ID"
     DEPLOYMENT_TO_KEEP="$CURRENT_ACTIVE"
+    echo "📋 Strategy: Deleting green (new) deployment, keeping blue (old)"
 fi
 
-delete_deployment_resources "$DEPLOYMENT_TO_CLEAN"
+echo "📋 Deployment to clean: $DEPLOYMENT_TO_CLEAN | Deployment to keep: $DEPLOYMENT_TO_KEEP"
 
-echo "Verifying cleanup for scope_id: $SCOPE_ID in namespace: $K8S_NAMESPACE"
+if ! delete_deployment_resources "$DEPLOYMENT_TO_CLEAN"; then
+    exit 1
+fi
+
+echo "🔍 Verifying cleanup for scope_id=$SCOPE_ID in namespace=$K8S_NAMESPACE..."
 
 # Get all unique deployment_ids for this scope_id
 ALL_DEPLOYMENT_IDS=$(kubectl get "$OBJECTS_TO_DELETE" -n "$K8S_NAMESPACE" \
@@ -32,12 +54,18 @@ ALL_DEPLOYMENT_IDS=$(kubectl get "$OBJECTS_TO_DELETE" -n "$K8S_NAMESPACE" \
 
 # Delete all deployment_ids except DEPLOYMENT_TO_KEEP
 if [ -n "$ALL_DEPLOYMENT_IDS" ]; then
+    EXTRA_COUNT=0
     while IFS= read -r EXTRA_DEPLOYMENT_ID; do
         if [ "$EXTRA_DEPLOYMENT_ID" != "$DEPLOYMENT_TO_KEEP" ]; then
+            echo "📝 Found orphaned deployment: $EXTRA_DEPLOYMENT_ID"
             delete_deployment_resources "$EXTRA_DEPLOYMENT_ID"
+            EXTRA_COUNT=$((EXTRA_COUNT + 1))
         fi
     done <<< "$ALL_DEPLOYMENT_IDS"
+    if [ "$EXTRA_COUNT" -gt 0 ]; then
+        echo "✅ Cleaned up $EXTRA_COUNT orphaned deployment(s)"
+    fi
 fi
 
-
-echo "Cleanup verification successful: Only deployment_id=$DEPLOYMENT_TO_KEEP remains for scope_id=$SCOPE_ID"
+echo "✨ Cluster cleanup completed successfully"
+echo "📋 Only deployment_id=$DEPLOYMENT_TO_KEEP remains for scope_id=$SCOPE_ID"

k8s/deployment/delete_ingress_finalizer

@@ -1,9 +1,24 @@
 #!/bin/bash
 
+echo "🔍 Checking for ingress finalizers to remove..."
+
 INGRESS_NAME=$(echo "$CONTEXT" | jq -r '"k-8-s-" + .scope.slug + "-" + (.scope.id | tostring) + "-" + .ingress_visibility')
+echo "📋 Ingress name: $INGRESS_NAME"
 
 # If the scope uses ingress, remove any finalizers attached to it
 if kubectl get ingress "$INGRESS_NAME" -n "$K8S_NAMESPACE" &>/dev/null; then
-  kubectl patch ingress "$INGRESS_NAME" -n "$K8S_NAMESPACE" -p '{"metadata":{"finalizers":[]}}' --type=merge
-fi
-# Do nothing if the scope does not use ingress (e.x: uses http route or has no network component)
+  echo "📝 Removing finalizers from ingress $INGRESS_NAME..."
+  if ! kubectl patch ingress "$INGRESS_NAME" -n "$K8S_NAMESPACE" -p '{"metadata":{"finalizers":[]}}' --type=merge; then
+    echo "❌ Failed to remove finalizers from ingress $INGRESS_NAME" >&2
+    echo "💡 Possible causes:" >&2
+    echo "   - Ingress was deleted while patching" >&2
+    echo "   - Insufficient permissions to patch ingress" >&2
+    echo "🔧 How to fix:" >&2
+    echo "   - Verify ingress still exists: kubectl get ingress $INGRESS_NAME -n $K8S_NAMESPACE" >&2
+    echo "   - Check RBAC permissions for patching ingress resources" >&2
+    exit 1
+  fi
+  echo "✅ Finalizers removed from ingress $INGRESS_NAME"
+else
+  echo "📋 Ingress $INGRESS_NAME not found, skipping finalizer removal"
+fi