Improve various error messages

Makefile

@@ -60,9 +60,9 @@ operator-local-dbg-gcp:
 
 # configure kubectl to point to the cluster specified in dev/config/cluster-[aws|gcp].yaml
 kubectl-aws:
-	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-aws.yaml) && eksctl utils write-kubeconfig --cluster="$$CORTEX_CLUSTER_NAME" --region="$$CORTEX_REGION" | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-aws.yaml) && eksctl utils write-kubeconfig --cluster="$$CORTEX_CLUSTER_NAME" --region="$$CORTEX_REGION" | (grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true)
 kubectl-gcp:
-	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-gcp.yaml) && gcloud container clusters get-credentials "$$CORTEX_CLUSTER_NAME" --zone "$$CORTEX_ZONE" --project "$$CORTEX_PROJECT" 2>&1 | grep -v "Fetching cluster" | grep -v "kubeconfig entry generated" || true
+	@eval $$(python3 ./manager/cluster_config_env.py ./dev/config/cluster-gcp.yaml) && gcloud container clusters get-credentials "$$CORTEX_CLUSTER_NAME" --project "$$CORTEX_PROJECT" --region "$$CORTEX_ZONE" 2> /dev/stdout 1> /dev/null | (grep -v "Fetching cluster" | grep -v "kubeconfig entry generated" || true)
 
 cluster-up-aws:
 	@$(MAKE) images-all-aws

cli/cmd/cluster_gcp.go

@@ -141,17 +141,24 @@ var _clusterGCPUpCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		bucketName := clusterconfig.GCPBucketName(*accessConfig.ClusterName, *accessConfig.Project, *accessConfig.Zone)
+		gkeClusterName := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", *clusterConfig.Project, *clusterConfig.Zone, clusterConfig.ClusterName)
+		bucketName := clusterconfig.GCPBucketName(clusterConfig.ClusterName, *clusterConfig.Project, *clusterConfig.Zone)
+
+		clusterExists, err := gcpClient.ClusterExists(gkeClusterName)
+		if err != nil {
+			exit.Error(err)
+		}
+		if clusterExists {
+			exit.Error(ErrorGCPClusterAlreadyExists(clusterConfig.ClusterName, *clusterConfig.Zone, *clusterConfig.Project))
+		}
+
 		err = gcpClient.CreateBucket(bucketName, gcp.ZoneToRegion(*accessConfig.Zone), true)
 		if err != nil {
 			exit.Error(err)
 		}
 
 		err = createGKECluster(clusterConfig, gcpClient)
 		if err != nil {
-			if errors.GetKind(err) != gcp.ErrClusterAlreadyExists {
-				gcpClient.DeleteBucket(bucketName)
-			}
 			exit.Error(err)
 		}
 
@@ -161,7 +168,6 @@ var _clusterGCPUpCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		gkeClusterName := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", *clusterConfig.Project, *clusterConfig.Zone, clusterConfig.ClusterName)
 		operatorLoadBalancerIP, err := getGCPOperatorLoadBalancerIP(gkeClusterName, gcpClient)
 		if err != nil {
 			exit.Error(errors.Append(err, fmt.Sprintf("\n\nyou can attempt to resolve this issue and configure your cli environment by running `cortex cluster info --configure-env %s`", _flagClusterGCPUpEnv)))
@@ -238,6 +244,16 @@ var _clusterGCPDownCmd = &cobra.Command{
 		}
 
 		gkeClusterName := fmt.Sprintf("projects/%s/locations/%s/clusters/%s", *accessConfig.Project, *accessConfig.Zone, *accessConfig.ClusterName)
+		bucketName := clusterconfig.GCPBucketName(*accessConfig.ClusterName, *accessConfig.Project, *accessConfig.Zone)
+
+		clusterExists, err := gcpClient.ClusterExists(gkeClusterName)
+		if err != nil {
+			exit.Error(err)
+		}
+		if !clusterExists {
+			gcpClient.DeleteBucket(bucketName) // silently try to delete the bucket in case it got left behind
+			exit.Error(ErrorGCPClusterDoesntExist(*accessConfig.ClusterName, *accessConfig.Zone, *accessConfig.Project))
+		}
 
 		// updating CLI env is best-effort, so ignore errors
 		operatorLoadBalancerIP, _ := getGCPOperatorLoadBalancerIP(gkeClusterName, gcpClient)
@@ -248,7 +264,6 @@ var _clusterGCPDownCmd = &cobra.Command{
 			prompt.YesOrExit(fmt.Sprintf("your cluster named \"%s\" in %s (zone: %s) will be spun down and all apis will be deleted, are you sure you want to continue?", *accessConfig.ClusterName, *accessConfig.Project, *accessConfig.Zone), "", "")
 		}
 
-		bucketName := clusterconfig.GCPBucketName(*accessConfig.ClusterName, *accessConfig.Project, *accessConfig.Zone)
 		fmt.Printf("￮ deleting bucket %s ", bucketName)
 		err = gcpClient.DeleteBucket(bucketName)
 		if err != nil {
@@ -518,6 +533,10 @@ func createGKECluster(clusterConfig *clusterconfig.GCPConfig, gcpClient *gcp.Cli
 		Cluster: &gkeClusterConfig,
 	})
 	if err != nil {
+		fmt.Print("\n\n")
+		if strings.Contains(errors.Message(err), "has no network named \"default\"") {
+			err = errors.Append(err, "\n\nyou can specify a different network be setting the `network` field in your cluster configuration file (see https://docs.cortex.dev)")
+		}
 		return err
 	}
 

cli/cmd/errors.go

@@ -75,6 +75,8 @@ const (
 	ErrShellCompletionNotSupported             = "cli.shell_completion_not_supported"
 	ErrNoTerminalWidth                         = "cli.no_terminal_width"
 	ErrDeployFromTopLevelDir                   = "cli.deploy_from_top_level_dir"
+	ErrGCPClusterAlreadyExists                 = "cli.gcp_cluster_already_exists"
+	ErrGCPClusterDoesntExist                   = "cli.gcp_cluster_doesnt_exist"
 )
 
 func ErrorInvalidProvider(providerStr string) error {
@@ -332,3 +334,17 @@ func ErrorDeployFromTopLevelDir(genericDirName string, providerType types.Provid
 		Message: fmt.Sprintf("cannot deploy from your %s directory - when deploying your API, cortex sends all files in your project directory (i.e. the directory which contains cortex.yaml) to your cluster (see https://docs.cortex.dev/v/%s/); therefore it is recommended to create a subdirectory for your project files", genericDirName, consts.CortexVersionMinor),
 	})
 }
+
+func ErrorGCPClusterAlreadyExists(clusterName string, zone string, project string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrGCPClusterAlreadyExists,
+		Message: fmt.Sprintf("there is already a cluster named \"%s\" in %s in the %s project", clusterName, zone, project),
+	})
+}
+
+func ErrorGCPClusterDoesntExist(clusterName string, zone string, project string) error {
+	return errors.WithStack(&errors.Error{
+		Kind:    ErrGCPClusterDoesntExist,
+		Message: fmt.Sprintf("there is no cluster named \"%s\" in %s in the %s project", clusterName, zone, project),
+	})
+}

cli/cmd/version.go

@@ -39,9 +39,11 @@ var _versionCmd = &cobra.Command{
 	Args:  cobra.NoArgs,
 	Run: func(cmd *cobra.Command, args []string) {
 		envName, err := getEnvFromFlag(_flagVersionEnv)
-		if err != nil {
+
+		if err != nil || envName == "" {
 			telemetry.Event("cli.version")
-			exit.Error(err)
+			fmt.Println("cli version: " + consts.CortexVersion)
+			return
 		}
 
 		env, err := ReadOrConfigureEnv(envName)

manager/debug.sh

@@ -26,7 +26,7 @@ if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTE
   exit 1
 fi
 
-eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | (grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true)
 out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi
 
 echo -n "gathering cluster data"

manager/debug_gcp.sh

@@ -21,8 +21,8 @@ CORTEX_VERSION_MINOR=master
 debug_out_path="$1"
 mkdir -p "$(dirname "$debug_out_path")"
 
-gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS > /dev/null 2>&1
-gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE > /dev/null 2>&1 # write both stderr and stdout to dev/null
+gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS 2> /dev/stdout 1> /dev/null | (grep -v "Activated service account credentials" || true)
+gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE 2> /dev/stdout 1> /dev/null | (grep -v "Fetching cluster" | grep -v "kubeconfig entry generated" || true)
 out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your iam user does not have access to this cluster"; exit 1; fi
 
 echo -n "gathering cluster data"

manager/info.sh

@@ -31,7 +31,7 @@ if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTE
   exit 1
 fi
 
-eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | (grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true)
 out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi
 
 operator_endpoint=$(get_operator_endpoint)

manager/info_gcp.sh

@@ -26,8 +26,8 @@ function get_api_load_balancer_endpoint() {
   kubectl -n=istio-system get service ingressgateway-apis -o json | tr -d '[:space:]' | sed 's/.*{\"ip\":\"\(.*\)\".*/\1/'
 }
 
-gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS > /dev/null 2>&1
-gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE > /dev/null 2>&1 # write both stderr and stdout to dev/null
+gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS 2> /dev/stdout 1> /dev/null | (grep -v "Activated service account credentials" || true)
+gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE 2> /dev/stdout 1> /dev/null | (grep -v "Fetching cluster" | grep -v "kubeconfig entry generated" || true)
 out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your iam user does not have access to this cluster"; exit 1; fi
 
 operator_endpoint=$(get_operator_endpoint)

manager/install.sh

@@ -97,8 +97,8 @@ function cluster_up_aws() {
 }
 
 function cluster_up_gcp() {
-  gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS > /dev/null 2>&1
-  gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE > /dev/null 2>&1 # write both stderr and stdout to dev/null
+  gcloud auth activate-service-account --key-file $GOOGLE_APPLICATION_CREDENTIALS 2> /dev/stdout 1> /dev/null | (grep -v "Activated service account credentials" || true)
+  gcloud container clusters get-credentials $CORTEX_CLUSTER_NAME --project $CORTEX_GCP_PROJECT --region $CORTEX_GCP_ZONE 2> /dev/stdout 1> /dev/null | (grep -v "Fetching cluster" | grep -v "kubeconfig entry generated" || true)
 
   start_pre_download_images
 
@@ -248,7 +248,7 @@ function check_eks() {
 }
 
 function write_kubeconfig() {
-  eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+  eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | (grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true)
   out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi
 }
 

manager/refresh.sh

@@ -26,7 +26,7 @@ if ! eksctl utils describe-stacks --cluster=$CORTEX_CLUSTER_NAME --region=$CORTE
   exit 1
 fi
 
-eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
+eksctl utils write-kubeconfig --cluster=$CORTEX_CLUSTER_NAME --region=$CORTEX_REGION | (grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true)
 out=$(kubectl get pods 2>&1 || true); if [[ "$out" == *"must be logged in to the server"* ]]; then echo "error: your aws iam user does not have access to this cluster; to grant access, see https://docs.cortex.dev/v/${CORTEX_VERSION_MINOR}/"; exit 1; fi
 
 kubectl get -n=default configmap cluster-config -o yaml >> cluster_configmap.yaml

pkg/lib/aws/s3.go

@@ -706,6 +706,10 @@ func (c *Client) DeleteS3Dir(bucket string, s3Dir string, continueIfFailure bool
 
 func (c *Client) DeleteS3Prefix(bucket string, prefix string, continueIfFailure bool) error {
 	err := c.S3BatchIterator(bucket, prefix, true, nil, func(objects []*s3.Object) (bool, error) {
+		if len(objects) == 0 {
+			return true, nil
+		}
+
 		deleteObjects := make([]*s3.ObjectIdentifier, len(objects))
 		for i, object := range objects {
 			deleteObjects[i] = &s3.ObjectIdentifier{Key: object.Key}

pkg/lib/gcp/errors.go

@@ -30,8 +30,6 @@ const (
 	ErrInvalidGCSPath              = "gcp.invalid_gcs_path"
 	ErrCredentialsFileEnvVarNotSet = "gcp.credentials_file_env_var_not_set"
 	ErrProjectIDMismatch           = "gcp.project_id_mismatch"
-
-	ErrClusterAlreadyExists = "gcp.cluster_already_exists"
 )
 
 func IsGCPError(err error) bool {
@@ -81,10 +79,3 @@ func ErrorProjectIDMismatch(credsFileProject string, providedProject string, cre
 		Message: fmt.Sprintf("the \"%s\" project was specified in your configuration, but the credentials file at %s (specified via $GOOGLE_APPLICATION_CREDENTIALS) is connected the the project named \"%s\"", providedProject, credsFilePath, credsFileProject),
 	})
 }
-
-func ErrorClusterAlreadyExists(clusterName string) error {
-	return errors.WithStack(&errors.Error{
-		Kind:    ErrClusterAlreadyExists,
-		Message: fmt.Sprintf("cluster %s already exists", clusterName),
-	})
-}

pkg/lib/gcp/gke.go

@@ -19,7 +19,6 @@ package gcp
 import (
 	"context"
 	"encoding/base64"
-	"fmt"
 
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	containerpb "google.golang.org/genproto/googleapis/container/v1"
@@ -80,11 +79,6 @@ func (c *Client) CreateCluster(req *containerpb.CreateClusterRequest) (*containe
 	if err != nil {
 		return nil, err
 	}
-	if exists, err := c.ClusterExists(fmt.Sprintf("%s/clusters/%s", req.Parent, req.Cluster.Name)); err != nil {
-		return nil, err
-	} else if exists {
-		return nil, ErrorClusterAlreadyExists(req.Cluster.Name)
-	}
 	resp, err := gke.CreateCluster(context.Background(), req)
 	if err != nil {
 		return nil, errors.WithStack(err)

pkg/lib/prompt/prompt.go

@@ -70,6 +70,9 @@ func Prompt(opts *Options) string {
 		if errors.Message(err) == "interrupted" {
 			exit.Error(ErrorUserCtrlC())
 		}
+		if strings.Contains(errors.Message(err), "not a terminal") {
+			err = errors.Append(err, "\n\nyou may be able to pass flags into this command to provide all required inputs and/or skip prompts (e.g. via `--yes`)")
+		}
 		exit.Error(err)
 	}
 

pkg/types/spec/utils.go

@@ -21,6 +21,7 @@ import (
 	"strconv"
 	"strings"
 
+	"github.com/cortexlabs/cortex/pkg/consts"
 	"github.com/cortexlabs/cortex/pkg/lib/aws"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/files"
@@ -162,12 +163,17 @@ func validateDirModels(
 
 	modelResources := make([]CuratedModelResource, len(modelNames))
 	for i, modelName := range modelNames {
+		modelNameWrapStr := modelName
+		if modelName == consts.SingleModelName {
+			modelNameWrapStr = ""
+		}
+
 		modelPrefix := filepath.Join(dirPrefix, modelName)
 		modelPrefix = s.EnsureSuffix(modelPrefix, "/")
 
 		modelStructureType := determineBaseModelStructure(modelDirPaths, modelPrefix)
 		if modelStructureType == userconfig.UnknownModelStructureType {
-			return nil, errors.Wrap(errorForPredictorType(modelPrefix, nil), modelName)
+			return nil, errors.Wrap(errorForPredictorType(modelPrefix, nil), modelNameWrapStr)
 		}
 
 		var versions []string
@@ -180,22 +186,22 @@ func validateDirModels(
 				for _, validator := range extraValidators {
 					err := validator(modelDirPaths, modelPrefix, pointer.String(versionedModelPrefix))
 					if err != nil {
-						return nil, errors.Wrap(err, modelName)
+						return nil, errors.Wrap(err, modelNameWrapStr)
 					}
 				}
 			}
 		} else {
 			for _, validator := range extraValidators {
 				err := validator(modelDirPaths, modelPrefix, nil)
 				if err != nil {
-					return nil, errors.Wrap(err, modelName)
+					return nil, errors.Wrap(err, modelNameWrapStr)
 				}
 			}
 		}
 
 		intVersions, err := slices.StringToInt64(versions)
 		if err != nil {
-			return nil, errors.Wrap(err, modelName)
+			return nil, errors.Wrap(err, modelNameWrapStr)
 		}
 
 		fullModelPath := ""
@@ -236,6 +242,11 @@ func validateModels(
 
 	modelResources := make([]CuratedModelResource, len(models))
 	for i, model := range models {
+		modelNameWrapStr := model.Name
+		if model.Name == consts.SingleModelName {
+			modelNameWrapStr = ""
+		}
+
 		modelPath := s.EnsureSuffix(model.Path, "/")
 
 		s3Path := strings.HasPrefix(model.Path, "s3://")
@@ -244,41 +255,41 @@ func validateModels(
 		if s3Path {
 			awsClientForBucket, err := aws.NewFromClientS3Path(model.Path, awsClient)
 			if err != nil {
-				return nil, errors.Wrap(err, model.Name)
+				return nil, errors.Wrap(err, modelNameWrapStr)
 			}
 
 			bucket, modelPrefix, err = aws.SplitS3Path(model.Path)
 			if err != nil {
-				return nil, errors.Wrap(err, model.Name)
+				return nil, errors.Wrap(err, modelNameWrapStr)
 			}
 			modelPrefix = s.EnsureSuffix(modelPrefix, "/")
 
 			s3Objects, err := awsClientForBucket.ListS3PathDir(modelPath, false, nil)
 			if err != nil {
-				return nil, errors.Wrap(err, model.Name)
+				return nil, errors.Wrap(err, modelNameWrapStr)
 			}
 			modelPaths = aws.ConvertS3ObjectsToKeys(s3Objects...)
 		}
 		if gcsPath {
 			bucket, modelPrefix, err = gcp.SplitGCSPath(model.Path)
 			if err != nil {
-				return nil, errors.Wrap(err, model.Name)
+				return nil, errors.Wrap(err, modelNameWrapStr)
 			}
 			modelPrefix = s.EnsureSuffix(modelPrefix, "/")
 
 			gcsObjects, err := gcpClient.ListGCSPathDir(modelPath, nil)
 			if err != nil {
-				return nil, errors.Wrap(err, model.Name)
+				return nil, errors.Wrap(err, modelNameWrapStr)
 			}
 			modelPaths = gcp.ConvertGCSObjectsToKeys(gcsObjects...)
 		}
 		if len(modelPaths) == 0 {
-			return nil, errors.Wrap(errorForPredictorType(modelPrefix, modelPaths), model.Name)
+			return nil, errors.Wrap(errorForPredictorType(modelPrefix, modelPaths), modelNameWrapStr)
 		}
 
 		modelStructureType := determineBaseModelStructure(modelPaths, modelPrefix)
 		if modelStructureType == userconfig.UnknownModelStructureType {
-			return nil, errors.Wrap(ErrorInvalidPythonModelPath(modelPath, []string{}), model.Name)
+			return nil, errors.Wrap(ErrorInvalidPythonModelPath(modelPath, []string{}), modelNameWrapStr)
 		}
 
 		var versions []string
@@ -291,22 +302,22 @@ func validateModels(
 				for _, validator := range extraValidators {
 					err := validator(modelPaths, modelPrefix, pointer.String(versionedModelPrefix))
 					if err != nil {
-						return nil, errors.Wrap(err, model.Name)
+						return nil, errors.Wrap(err, modelNameWrapStr)
 					}
 				}
 			}
 		} else {
 			for _, validator := range extraValidators {
 				err := validator(modelPaths, modelPrefix, nil)
 				if err != nil {
-					return nil, errors.Wrap(err, model.Name)
+					return nil, errors.Wrap(err, modelNameWrapStr)
 				}
 			}
 		}
 
 		intVersions, err := slices.StringToInt64(versions)
 		if err != nil {
-			return nil, errors.Wrap(err, model.Name)
+			return nil, errors.Wrap(err, modelNameWrapStr)
 		}
 
 		var signatureKey *string