From 2befaae75f1bcb3e676ffcd943516149108c4399 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:08:27 -0700 Subject: [PATCH 1/7] Add note regarding private operator load balancer initializing --- manager/install.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/manager/install.sh b/manager/install.sh index ec1a6be41c..5d096057e1 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -202,6 +202,12 @@ function main() { python update_cli_config.py "/.cortex/cli.yaml" "$CORTEX_ENV_NAME" "$operator_endpoint" "$CORTEX_AWS_ACCESS_KEY_ID" "$CORTEX_AWS_SECRET_ACCESS_KEY" echo "✓" + if [ "$arg1" != "--update" ] && [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then + echo -e "\ncortex is ready! (it may take a few minutes for your private operator load balancer to finish initializing)" + else + echo -e "\ncortex is ready!" + fi + echo -e "\ncortex is ready!" } From 79eb6f114a73537df25421f557b97c6e8589e693 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:08:38 -0700 Subject: [PATCH 2/7] Add 30 minute cluster validation timeout --- manager/install.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/manager/install.sh b/manager/install.sh index 5d096057e1..3619d48c9a 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -275,6 +275,8 @@ function setup_istio() { function validate_cortex() { set +e + validation_start_time="$(date +%s)" + echo -n "￮ waiting for load balancers " operator_load_balancer="waiting" @@ -284,6 +286,13 @@ function validate_cortex() { operator_endpoint="" while true; do + # 30 minute timeout + now="$(date +%s)" + if [ "$now" -ge "$(($validation_start_time+1800))" ]; then + echo -e "\n\ntimeout has occurred when validating your cortex cluster" + exit 1 + fi + echo -n "." sleep 3 From 63d7cdd76833f66e96398d2c90b9e9b4f9fa8d96 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:10:56 -0700 Subject: [PATCH 3/7] Update message --- manager/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/install.sh b/manager/install.sh index 3619d48c9a..e817d6a5fb 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -203,7 +203,7 @@ function main() { echo "✓" if [ "$arg1" != "--update" ] && [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then - echo -e "\ncortex is ready! (it may take a few minutes for your private operator load balancer to finish initializing)" + echo -e "\ncortex is ready! (it may take a few minutes for your private operator load balancer to finish initializing, but you may now set up VPC Peering)" else echo -e "\ncortex is ready!" fi From 1321a64836d43a6ffa88cbd77580428adf888eb9 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:13:48 -0700 Subject: [PATCH 4/7] Add operator curl timeout --- manager/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/install.sh b/manager/install.sh index e817d6a5fb..cf219f84a4 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -330,7 +330,7 @@ function validate_cortex() { if [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then if [ "$operator_endpoint_reachable" != "ready" ]; then - if ! curl $operator_endpoint >/dev/null 2>&1; then + if ! curl --max-time 3 $operator_endpoint >/dev/null 2>&1; then continue fi operator_endpoint_reachable="ready" From 981c05dde10202206405e928b4c7130ca9fb33c1 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:39:59 -0700 Subject: [PATCH 5/7] Remove extra print --- manager/install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/manager/install.sh b/manager/install.sh index cf219f84a4..c3ce700616 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -207,8 +207,6 @@ function main() { else echo -e "\ncortex is ready!" fi - - echo -e "\ncortex is ready!" } function setup_configmap() { From 701ce284486686d2d25f565afa27f225024e8154 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:40:29 -0700 Subject: [PATCH 6/7] Update install.sh --- manager/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manager/install.sh b/manager/install.sh index c3ce700616..7dc05adf25 100755 --- a/manager/install.sh +++ b/manager/install.sh @@ -202,7 +202,7 @@ function main() { python update_cli_config.py "/.cortex/cli.yaml" "$CORTEX_ENV_NAME" "$operator_endpoint" "$CORTEX_AWS_ACCESS_KEY_ID" "$CORTEX_AWS_SECRET_ACCESS_KEY" echo "✓" - if [ "$arg1" != "--update" ] && [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internet-facing" ]; then + if [ "$arg1" != "--update" ] && [ "$CORTEX_OPERATOR_LOAD_BALANCER_SCHEME" == "internal" ]; then echo -e "\ncortex is ready! (it may take a few minutes for your private operator load balancer to finish initializing, but you may now set up VPC Peering)" else echo -e "\ncortex is ready!" From d60e5fa83420a25bf5686e941f53f0976f57f4a9 Mon Sep 17 00:00:00 2001 From: David Eliahu Date: Sun, 3 May 2020 15:44:22 -0700 Subject: [PATCH 7/7] Update cluster.go --- cli/cmd/cluster.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go index 93eef840c5..629dcc1480 100644 --- a/cli/cmd/cluster.go +++ b/cli/cmd/cluster.go @@ -153,7 +153,7 @@ var _upCmd = &cobra.Command{ exit.Error(err) } if exitCode == nil || *exitCode != 0 { - helpStr := "\nDebugging tips (may not apply to this error):" + helpStr := "\nDebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf("\n* if your cluster started spinning up but was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", *clusterConfig.Region) helpStr += fmt.Sprintf("\n* if your cluster started spinning up, please ensure that your CloudFormation stacks for this cluster have been fully deleted before trying to spin up this cluster again: https://console.aws.amazon.com/cloudformation/home?region=%s#/stacks?filteringText=-%s-", *clusterConfig.Region, clusterConfig.ClusterName) fmt.Println(helpStr) @@ -219,7 +219,7 @@ var _updateCmd = &cobra.Command{ exit.Error(err) } if exitCode == nil || *exitCode != 0 { - helpStr := "\nDebugging tips (may not apply to this error):" + helpStr := "\nDebugging tips (may or may not apply to this error):" helpStr += fmt.Sprintf("\n* if your cluster was unable to provision instances, additional error information may be found in the activity history of your cluster's autoscaling groups (select each autoscaling group and click the \"Activity History\" tab): https://console.aws.amazon.com/ec2/autoscaling/home?region=%s#AutoScalingGroups:", *clusterConfig.Region) fmt.Println(helpStr) exit.Error(ErrorClusterUpdate(out + helpStr))