[UI] Fix Trial Logs when Kubernetes Job Fails (#2164)

* [UI] Fix Trial Logs when Kubernetes Job Fails

* Return error when Pod is in the Pending state

pkg/new-ui/v1beta1/backend.go

@@ -733,11 +733,23 @@ func fetchMasterPodName(clientset *kubernetes.Clientset, trial *trialsv1beta1.Tr
 		field to "true" in the Experiment definition. If this error persists then the Pod's logs are not currently
 		persisted in the cluster.`)
 	}
-	if len(podList.Items) > 1 {
-		return "", errors.New("More than one master replica found")
+
+	// If Pod is Running or Succeeded Pod, return it.
+	for _, pod := range podList.Items {
+		if pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodRunning {
+			return pod.Name, nil
+		}
+	}
+
+	// Otherwise, return the first Failed Pod.
+	for _, pod := range podList.Items {
+		if pod.Status.Phase == corev1.PodFailed {
+			return pod.Name, nil
+		}
 	}
 
-	return podList.Items[0].Name, nil
+	// Otherwise, return error since Pod is in the Pending state.
+	return "", errors.New("Failed to get logs for this Trial. Pod is in the Pending or Unknown state.")
 }
 
 // fetchPodLogs returns logs of a pod for the given job name and namespace