Surface pod creation errors

cli/cmd/get.go

@@ -40,13 +40,18 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
 	libtime "github.com/cortexlabs/cortex/pkg/lib/time"
 	"github.com/cortexlabs/cortex/pkg/lib/urls"
+	"github.com/cortexlabs/cortex/pkg/operator/config"
 	"github.com/cortexlabs/cortex/pkg/operator/schema"
 	"github.com/cortexlabs/cortex/pkg/types"
 	"github.com/cortexlabs/cortex/pkg/types/metrics"
 	"github.com/cortexlabs/cortex/pkg/types/spec"
 	"github.com/cortexlabs/cortex/pkg/types/status"
 	"github.com/cortexlabs/cortex/pkg/types/userconfig"
+
 	"github.com/spf13/cobra"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	klabels "k8s.io/apimachinery/pkg/labels"
 )
 
 const (
@@ -348,13 +353,30 @@ func apiTable(apis []spec.API, statuses []status.Status, allMetrics []metrics.Me
 	var total5XX int
 
 	for i, api := range apis {
+		deploy, _ := config.K8s.GetDeployment(api.DeploymentID)
+		msg := ""
+		if deploy != nil {
+			labels := deploy.Spec.Template.GetLabels()
+			pods, _ := config.K8s.ListPods(&metav1.ListOptions{LabelSelector: klabels.SelectorFromSet(labels).String()})
+			if len(pods) >= 1 {
+				for _, pod := range pods {
+					if pod.Status.Phase != v1.PodRunning {
+						if pod.Status.ContainerStatuses[0].State.Waiting != nil {
+							msg = pod.Status.ContainerStatuses[0].State.Waiting.Reason
+							break
+						}
+					}
+				}
+			}
+		}
+
 		metrics := allMetrics[i]
 		status := statuses[i]
 		lastUpdated := time.Unix(api.LastUpdated, 0)
 		rows = append(rows, []interface{}{
 			envNames[i],
 			api.Name,
-			status.Message(),
+			status.Message(msg),
 			status.Updated.Ready,
 			status.Stale.Ready,
 			status.Requested,

cli/cmd/predict.go

@@ -82,7 +82,7 @@ var _predictCmd = &cobra.Command{
 
 		totalReady := apiRes.Status.Updated.Ready + apiRes.Status.Stale.Ready
 		if totalReady == 0 {
-			exit.Error(ErrorAPINotReady(apiName, apiRes.Status.Message()))
+			exit.Error(ErrorAPINotReady(apiName, apiRes.Status.Message("")))
 		}
 
 		predictResponse, err := makePredictRequest(apiEndpoint, jsonPath)

pkg/operator/operator/cron.go

@@ -124,8 +124,8 @@ func operatorTelemetry() error {
 	var totalInstancePrice float64
 	var totalInstancePriceIfOnDemand float64
 	for _, info := range instanceInfos {
-		totalInstancePrice += ((info.Price + apiEBSPrice) * float64(info.Count))
-		totalInstancePriceIfOnDemand += ((info.OnDemandPrice + apiEBSPrice) * float64(info.Count))
+		totalInstancePrice += (info.Price + apiEBSPrice) * float64(info.Count)
+		totalInstancePriceIfOnDemand += (info.OnDemandPrice + apiEBSPrice) * float64(info.Count)
 	}
 
 	fixedPrice := clusterFixedPrice()

pkg/types/status/status.go

@@ -41,7 +41,10 @@ type SubReplicaCounts struct {
 	Unknown      int32 `json:"unknown"`
 }
 
-func (status *Status) Message() string {
+func (status *Status) Message(msg string) string {
+	if msg != "" {
+		return msg
+	}
 	return status.Code.Message()
 }