From 51ea24431d678ad941d7e849ab846cb13209d41e Mon Sep 17 00:00:00 2001 From: vishal Date: Tue, 27 Jul 2021 16:37:10 -0400 Subject: [PATCH 1/2] Address sentry errors --- cli/cluster/errors.go | 2 ++ cli/cmd/cluster.go | 6 ++++-- pkg/operator/endpoints/errors.go | 11 +++++++++++ pkg/operator/endpoints/middleware.go | 2 +- pkg/operator/resources/asyncapi/queue_metrics.go | 7 ++++--- python/client/cortex/client.py | 16 ++++++---------- 6 files changed, 28 insertions(+), 16 deletions(-) diff --git a/cli/cluster/errors.go b/cli/cluster/errors.go index 7a65f95c66..b370f349e8 100644 --- a/cli/cluster/errors.go +++ b/cli/cluster/errors.go @@ -63,6 +63,8 @@ func ErrorFailedToConnectOperator(originalError error, envName string, operatorU msg += "\nif you have a cluster running:\n" msg += fmt.Sprintf(" → run `cortex cluster info --configure-env %s` to update your environment (include `--config ` if you have a cluster configuration file)\n", envName) msg += fmt.Sprintf(" → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortex.dev/v/%s/)\n", consts.CortexVersionMinor) + msg += fmt.Sprintf(" → confirm that the ip address of this machine falls within the CIDR ranges specified in `operator_load_balancer_cidr_whitelist`") + } return errors.WithStack(&errors.Error{ diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index f89ba386f3..da2385664a 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -235,14 +235,14 @@ var _clusterUpCmd = &cobra.Command{ exit.Error(err) } if exitCode == nil || *exitCode != 0 { - out = filterEKSCTLOutput(out) + out = s.LastNChars(filterEKSCTLOutput(out), 8192) // get the last 8192 characters because that is the sentry message limit eksCluster, err := awsClient.EKSClusterOrNil(clusterConfig.ClusterName) if err != nil { helpStr := "\ndebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf("\n* if your cluster started spinning up but was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity\" or \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", clusterConfig.Region) helpStr += "\n* if your cluster started spinning up, please run `cortex cluster down` to delete the cluster before trying to create this cluster again" fmt.Println(helpStr) - exit.Error(ErrorClusterUp(out + helpStr)) + exit.Error(ErrorClusterUp(out)) } // the cluster never started spinning up @@ -398,6 +398,8 @@ var _clusterConfigureCmd = &cobra.Command{ exit.Error(err) } if exitCode == nil || *exitCode != 0 { + out = s.LastNChars(out, 8192) // get the last 8192 characters because that is the sentry message limit + helpStr := "\ndebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf( "\n* if your cluster was unable to provision/remove/scale some nodegroups, additional error information may be found in the description of your cloudformation stack (https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks)"+ diff --git a/pkg/operator/endpoints/errors.go b/pkg/operator/endpoints/errors.go index a005323489..b23b31aaaa 100644 --- a/pkg/operator/endpoints/errors.go +++ b/pkg/operator/endpoints/errors.go @@ -53,6 +53,17 @@ func ErrorHeaderMissing(header string) error { }) } +func ErrorAuthHeaderMissing(header, host, url string) error { + return errors.WithStack(&errors.Error{ + Kind: ErrHeaderMissing, + Message: fmt.Sprintf("missing %s header", header), + Metadata: map[string]string{ + "host": host, + "url": url, + }, + }) +} + func ErrorHeaderMalformed(header string) error { return errors.WithStack(&errors.Error{ Kind: ErrHeaderMalformed, diff --git a/pkg/operator/endpoints/middleware.go b/pkg/operator/endpoints/middleware.go index ae2c98e7ca..8220abe9a5 100644 --- a/pkg/operator/endpoints/middleware.go +++ b/pkg/operator/endpoints/middleware.go @@ -64,7 +64,7 @@ func AWSAuthMiddleware(next http.Handler) http.Handler { authHeader := r.Header.Get(consts.AuthHeader) if authHeader == "" { - respondError(w, r, ErrorHeaderMissing(consts.AuthHeader)) + respondError(w, r, ErrorAuthHeaderMissing(consts.AuthHeader, r.Host, r.RequestURI)) return } diff --git a/pkg/operator/resources/asyncapi/queue_metrics.go b/pkg/operator/resources/asyncapi/queue_metrics.go index b5692b54f3..e6e2119d36 100644 --- a/pkg/operator/resources/asyncapi/queue_metrics.go +++ b/pkg/operator/resources/asyncapi/queue_metrics.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/sqs" "github.com/cortexlabs/cortex/pkg/config" + "github.com/cortexlabs/cortex/pkg/lib/errors" "github.com/cortexlabs/cortex/pkg/types/userconfig" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -74,7 +75,7 @@ func updateQueueLengthMetricsFn(apiName, queueURL string) func() error { output, err := sqsClient.GetQueueAttributesWithContext(ctx, input) if err != nil { - return err + return errors.WithStack(err) } visibleMessagesStr := output.Attributes["ApproximateNumberOfMessages"] @@ -82,12 +83,12 @@ func updateQueueLengthMetricsFn(apiName, queueURL string) func() error { visibleMessages, err := strconv.ParseFloat(*visibleMessagesStr, 64) if err != nil { - return err + return errors.WithStack(err) } invisibleMessages, err := strconv.ParseFloat(*invisibleMessagesStr, 64) if err != nil { - return err + return errors.WithStack(err) } activeGauge.WithLabelValues(apiName).Set(invisibleMessages) diff --git a/python/client/cortex/client.py b/python/client/cortex/client.py index bf09da80b5..9a8a5882f1 100644 --- a/python/client/cortex/client.py +++ b/python/client/cortex/client.py @@ -15,31 +15,27 @@ import json import os import shutil -import subprocess -import sys -import threading import time import yaml -from pathlib import Path -from typing import Optional, List, Dict, Any +from typing import List, Dict, Any from cortex import util -from cortex.binary import run_cli, get_cli_path +from cortex.binary import run_cli from cortex.telemetry import sentry_wrapper class Client: @sentry_wrapper - def __init__(self, env: Dict): + def __init__(self, env_config: Dict): """ A client to deploy and manage APIs in the specified environment. Args: - env: Environment config + env_config: Environment config """ - self.env = env - self.env_name = env["name"] + self.env = env_config + self.env_name = env_config["name"] # CORTEX_VERSION_MINOR @sentry_wrapper From 26612a1fbbe9edf639b9b15390f0a5a599795b5d Mon Sep 17 00:00:00 2001 From: vishal Date: Wed, 28 Jul 2021 20:50:28 -0400 Subject: [PATCH 2/2] Address PR comments --- cli/cluster/errors.go | 1 - python/client/cortex/client.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/cluster/errors.go b/cli/cluster/errors.go index b370f349e8..8eb63cb4a9 100644 --- a/cli/cluster/errors.go +++ b/cli/cluster/errors.go @@ -64,7 +64,6 @@ func ErrorFailedToConnectOperator(originalError error, envName string, operatorU msg += fmt.Sprintf(" → run `cortex cluster info --configure-env %s` to update your environment (include `--config ` if you have a cluster configuration file)\n", envName) msg += fmt.Sprintf(" → if you set `operator_load_balancer_scheme: internal` in your cluster configuration file, your CLI must run from within a VPC that has access to your cluster's VPC (see https://docs.cortex.dev/v/%s/)\n", consts.CortexVersionMinor) msg += fmt.Sprintf(" → confirm that the ip address of this machine falls within the CIDR ranges specified in `operator_load_balancer_cidr_whitelist`") - } return errors.WithStack(&errors.Error{ diff --git a/python/client/cortex/client.py b/python/client/cortex/client.py index 9a8a5882f1..14905403e7 100644 --- a/python/client/cortex/client.py +++ b/python/client/cortex/client.py @@ -29,6 +29,8 @@ class Client: def __init__(self, env_config: Dict): """ A client to deploy and manage APIs in the specified environment. + This constructor is not meant to be invoked directly. + Use `cortex.client()` and `cortex.new_client()` to initialize a new cortex client. Args: env_config: Environment config