feat: improve smoke tests visibility (IN-1101) (#250)

feat: improve smoke tests visibility (IN-1101)

src/jobs/e2e/collect-e2e-logs.yml

@@ -0,0 +1,118 @@
+description: Prepare an existing environment for e2e tests
+
+parameters:
+    cluster:
+      type: string
+      description: Name of the cluster in which the environment exists
+      default: "cm4-vf-dev-br-2-0-p1"
+    e2e-env-name:
+      type: string
+      description: Name of the environment to collect logs from
+    env-name-path:
+      type: string
+      description: Path to the env_name file
+      default: "/home/circleci/voiceflow/env_name.txt"
+    executor:
+     description: Executor to run the command on
+     type: executor
+     default: default-executor
+executor: << parameters.executor >>
+steps:
+  - install-vfcli:
+      init-cluster: << parameters.cluster >>
+  - restore_cache:
+      key: env_name_cache-{{ .Environment.CIRCLE_PROJECT_REPONAME }}-{{ .Environment.CIRCLE_WORKFLOW_ID }}
+  - run:
+      name: Create directories
+      environment:
+        LOG_DIR: &log_dir  /tmp/logs-<< parameters.e2e-env-name >>
+        COMPONENT_LOG_DIR: &component_log_dir component-logs
+        KUBE_STATE_DIR: &kube_state_dir kubernetes-state
+      command: |
+        mkdir -p "${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}"
+        mkdir -p "${LOG_DIR:?}/${KUBE_STATE_DIR:?}"
+  - run:
+      name: Gather Kubernetes state before run
+      environment:
+        LOG_DIR: *log_dir
+        KUBE_STATE_DIR: *kube_state_dir
+      command: |
+        echo "Contents of << parameters.env-name-path >>:"
+        cat << parameters.env-name-path >>
+        if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >> )" != "null" ]; then
+                DEV_ENV_NAME=$(cat << parameters.env-name-path >> )
+            else
+                DEV_ENV_NAME=<< parameters.e2e-env-name >>
+            fi
+        # Gather summary state of all pods in the namespace
+        echo "Gathering Kubernetes state before run for env $DEV_ENV_NAME"
+        kubectl get pods -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-before-run.log"
+  - run:
+      name: Gather Logs
+      environment:
+        LOG_DIR:  *log_dir
+        COMPONENT_LOG_DIR: *component_log_dir
+      background: true
+      command: |
+        function capture_logs() {
+            if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >> )" != "null" ]; then
+                DEV_ENV_NAME=$(cat << parameters.env-name-path >> )
+            else
+                DEV_ENV_NAME=<< parameters.e2e-env-name >>
+            fi
+            echo "Capturing logs for environment $DEV_ENV_NAME"
+            # Read components into an array directly from the command output
+            components=($(vfcli component list -n "${DEV_ENV_NAME:?}" | awk 'NR>3 {print $1}'))
+            # Iterate over the first n-1 components.Process log collection in parallel as background processes
+            for ((i = 0; i < ${#components[@]} - 1; i++)); do
+                component=${components[$i]}
+                echo "Capturing logs for component $component"
+                stern -n "${DEV_ENV_NAME:?}" -l "app.kubernetes.io/name=$component" >>"${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}/$component.log" --since 5m &
+            done
+            # Handle the last component separately to introduce blocking.If all components' log collection is done as non blocking tasks,
+            # the circleci step will terminate.So having last component as blocking ensures the collect logs step continues executing
+            last_component=${components[${#components[@]}-1]}
+            echo "Capturing logs for last component $last_component"
+            stern -n "${DEV_ENV_NAME:?}" -l "app.kubernetes.io/name=$last_component" >>"${LOG_DIR:?}/${COMPONENT_LOG_DIR:?}/$last_component.log" --since 5m
+        }
+        capture_logs
+
+  - run:
+     name: Wait for smoke-tests job to complete
+     command: |
+      # The waiter job keeps looping through to check if the smoke-tests job has been completed
+      while [[ $(curl --location --request GET "https://circleci.com/api/v2/workflow/$CIRCLE_WORKFLOW_ID/job" --header "Circle-Token: $CIRCLE_TOKEN"| jq -r '.items[]|select(.name == "vfcommon/run-smoke-tests")|.status' | grep -c "running") -gt 0 ]]
+        do
+          sleep 5
+        done
+  - run: echo "All required jobs have now completed"
+  - run:
+      name: Gather Kubernetes state after run
+      environment:
+        LOG_DIR: *log_dir
+        KUBE_STATE_DIR: *kube_state_dir
+      command: |
+        if [ -f << parameters.env-name-path >> ] && [ "$(cat << parameters.env-name-path >> )" != "null" ]; then
+              DEV_ENV_NAME=$(cat << parameters.env-name-path >> )
+          else
+              DEV_ENV_NAME=<< parameters.e2e-env-name >>
+          fi
+        # Read components into an array directly from the command output
+        components=($(vfcli component list -n "${DEV_ENV_NAME:?}" | awk 'NR>3 {print $1}'))
+        # Gather summary state of all pods in the namespace
+        kubectl get pods -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/pods-summary-state-after-run.log"
+        # Gather detailed state of all pods in the namespace
+        for ((i = 0; i < ${#components[@]} - 1; i++)); do
+            component=${components[$i]}
+            echo "Capturing logs for component $component"
+            kubectl describe pod $component -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/${component}-k8-state.log" &
+         done
+         # Handle the last component separately to introduce blocking.If all components' log collection is done as non blocking tasks,
+         # the circleci step will terminate.So having last component as blocking ensures the collect logs step continues executing
+         last_component=${components[${#components[@]}-1]}
+         kubectl describe pod $component -n $DEV_ENV_NAME >> "${LOG_DIR:?}/${KUBE_STATE_DIR:?}/${last_component}-k8-state.log"
+
+  - store_artifacts:
+      name: Store Logs
+      path: *log_dir
+      destination: logs