Operator refactor (#759)

cortexlabs · Jan 18, 2020 · 5decedc · 5decedc
1 parent c9f22b6
commit 5decedc
Show file tree

Hide file tree

Showing 242 changed files with 6,028 additions and 10,800 deletions.
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -10,6 +10,6 @@ checklist:
 - [ ] update the docs and add any new files to `summary.md`
 - [ ] merge to master
 - [ ] cherry-pick into release branches if applicable
-- [ ] check that the docs are up-to-date in [gitbook](https://www.cortex.dev/v/master)
+- [ ] check that the docs are up-to-date in [gitbook](https://cortex.dev/v/master)
 - [ ] alert the dev team if the dev environment changed
 - [ ] delete the branch once it's merged
diff --git a/Makefile b/Makefile
@@ -26,7 +26,6 @@ devstart:
 
 kubectl:
 	@eval $$(python ./manager/cluster_config_env.py ./dev/config/cluster.yaml) && eksctl utils write-kubeconfig --cluster="$$CORTEX_CLUSTER_NAME" --region="$$CORTEX_REGION" | grep -v "saved kubeconfig as" | grep -v "using region" | grep -v "eksctl version" || true
-	@kubectl config set-context --current --namespace=cortex >/dev/null
 
 cluster-up:
 	@$(MAKE) registry-all
@@ -52,12 +51,9 @@ cluster-update:
 	@kill $(shell pgrep -f rerun) >/dev/null 2>&1 || true
 	@./bin/cortex -c=./dev/config/cluster.yaml cluster update
 
-cortex-uninstall:
-	@./dev/uninstall_cortex.sh
-
 operator-stop:
 	@$(MAKE) kubectl
-	@kubectl delete --namespace=cortex --ignore-not-found=true deployment operator
+	@kubectl delete --namespace=default --ignore-not-found=true deployment operator
 
 # Docker images
 

diff --git a/README.md b/README.md
@@ -6,7 +6,7 @@ Cortex is an open source platform for deploying machine learning models as produ
 
 <!-- Delete on release branches -->
 <!-- CORTEX_VERSION_README_MINOR -->
-[install](https://www.cortex.dev/install) • [tutorial](https://www.cortex.dev/iris-classifier) • [docs](https://www.cortex.dev) • [examples](https://github.com/cortexlabs/cortex/tree/0.12/examples) • [we're hiring](https://angel.co/cortex-labs-inc/jobs) • [email us](mailto:hello@cortex.dev) • [chat with us](https://gitter.im/cortexlabs/cortex)<br><br>
+[install](https://cortex.dev/install) • [tutorial](https://cortex.dev/iris-classifier) • [docs](https://cortex.dev) • [examples](https://github.com/cortexlabs/cortex/tree/0.12/examples) • [we're hiring](https://angel.co/cortex-labs-inc/jobs) • [email us](mailto:hello@cortex.dev) • [chat with us](https://gitter.im/cortexlabs/cortex)<br><br>
 
 <!-- Set header Cache-Control=no-cache on the S3 object metadata (see https://help.github.com/en/articles/about-anonymized-image-urls) -->
 ![Demo](https://d1zqebknpdh033.cloudfront.net/demo/gif/v0.12.gif)

diff --git a/build/lint.sh b/build/lint.sh
@@ -75,7 +75,7 @@ output=$(cd "$ROOT" && find . -type f \
 ! -path "./.circleci/*" \
 ! -path "./.git/*" \
 ! -name LICENSE \
-! -name requirements.txt \
+! -name "*requirements.txt" \
 ! -name "go.*" \
 ! -name "*.md" \
 ! -name ".*" \

diff --git a/cli/cmd/cluster.go b/cli/cmd/cluster.go
@@ -23,7 +23,6 @@ import (
 	"os"
 	"time"
 
-	"github.com/cortexlabs/cortex/pkg/lib/clusterconfig"
 	cr "github.com/cortexlabs/cortex/pkg/lib/configreader"
 	"github.com/cortexlabs/cortex/pkg/lib/errors"
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
@@ -32,42 +31,43 @@ import (
 	"github.com/cortexlabs/cortex/pkg/lib/prompt"
 	"github.com/cortexlabs/cortex/pkg/lib/table"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
-	"github.com/cortexlabs/cortex/pkg/operator/api/schema"
+	"github.com/cortexlabs/cortex/pkg/operator/schema"
+	"github.com/cortexlabs/cortex/pkg/types/clusterconfig"
 	"github.com/spf13/cobra"
 )
 
-var flagClusterConfig string
-var flagDebug bool
+var _flagClusterConfig string
+var _flagDebug bool
 
 func init() {
-	addClusterConfigFlag(updateCmd)
-	addEnvFlag(updateCmd)
-	clusterCmd.AddCommand(updateCmd)
+	addClusterConfigFlag(_updateCmd)
+	addEnvFlag(_updateCmd)
+	_clusterCmd.AddCommand(_updateCmd)
 
-	addClusterConfigFlag(infoCmd)
-	addEnvFlag(infoCmd)
-	infoCmd.PersistentFlags().BoolVarP(&flagDebug, "debug", "d", false, "save the current cluster state to a file")
-	clusterCmd.AddCommand(infoCmd)
+	addClusterConfigFlag(_infoCmd)
+	addEnvFlag(_infoCmd)
+	_infoCmd.PersistentFlags().BoolVarP(&_flagDebug, "debug", "d", false, "save the current cluster state to a file")
+	_clusterCmd.AddCommand(_infoCmd)
 
-	addClusterConfigFlag(upCmd)
-	addEnvFlag(upCmd)
-	clusterCmd.AddCommand(upCmd)
+	addClusterConfigFlag(_upCmd)
+	addEnvFlag(_upCmd)
+	_clusterCmd.AddCommand(_upCmd)
 
-	addClusterConfigFlag(downCmd)
-	clusterCmd.AddCommand(downCmd)
+	addClusterConfigFlag(_downCmd)
+	_clusterCmd.AddCommand(_downCmd)
 }
 
 func addClusterConfigFlag(cmd *cobra.Command) {
-	cmd.PersistentFlags().StringVarP(&flagClusterConfig, "config", "c", "", "path to a cluster configuration file")
-	cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, configFileExts)
+	cmd.PersistentFlags().StringVarP(&_flagClusterConfig, "config", "c", "", "path to a cluster configuration file")
+	cmd.PersistentFlags().SetAnnotation("config", cobra.BashCompFilenameExt, _configFileExts)
 }
 
-var clusterCmd = &cobra.Command{
+var _clusterCmd = &cobra.Command{
 	Use:   "cluster",
 	Short: "manage a cluster",
 }
 
-var upCmd = &cobra.Command{
+var _upCmd = &cobra.Command{
 	Use:   "up",
 	Short: "spin up a cluster",
 	Args:  cobra.NoArgs,
@@ -79,7 +79,7 @@ var upCmd = &cobra.Command{
 		}
 
 		promptForEmail()
-		awsCreds, err := getAWSCredentials(flagClusterConfig)
+		awsCreds, err := getAWSCredentials(_flagClusterConfig)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -99,7 +99,7 @@ var upCmd = &cobra.Command{
 	},
 }
 
-var updateCmd = &cobra.Command{
+var _updateCmd = &cobra.Command{
 	Use:   "update",
 	Short: "update a cluster",
 	Args:  cobra.NoArgs,
@@ -110,7 +110,7 @@ var updateCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		awsCreds, err := getAWSCredentials(flagClusterConfig)
+		awsCreds, err := getAWSCredentials(_flagClusterConfig)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -132,7 +132,7 @@ var updateCmd = &cobra.Command{
 	},
 }
 
-var infoCmd = &cobra.Command{
+var _infoCmd = &cobra.Command{
 	Use:   "info",
 	Short: "get information about a cluster",
 	Args:  cobra.NoArgs,
@@ -142,12 +142,12 @@ var infoCmd = &cobra.Command{
 		if err := checkDockerRunning(); err != nil {
 			exit.Error(err)
 		}
-		awsCreds, err := getAWSCredentials(flagClusterConfig)
+		awsCreds, err := getAWSCredentials(_flagClusterConfig)
 		if err != nil {
 			exit.Error(err)
 		}
 
-		if flagDebug {
+		if _flagDebug {
 			accessConfig, err := getClusterAccessConfig()
 			if err != nil {
 				exit.Error(err)
@@ -186,28 +186,28 @@ var infoCmd = &cobra.Command{
 
 		httpResponse, err := HTTPGet("/info")
 		if err != nil {
-			fmt.Println(clusterConfig.UserFacingString())
-			fmt.Println("\n" + errors.Wrap(err, "unable to connect to operator").Error())
+			fmt.Println(clusterConfig.UserStr())
+			fmt.Println("\n" + errors.Message(err, "unable to connect to operator"))
 			return
 		}
 		var infoResponse schema.InfoResponse
 		err = json.Unmarshal(httpResponse, &infoResponse)
 		if err != nil {
-			fmt.Println(clusterConfig.UserFacingString())
-			fmt.Println("\n" + errors.Wrap(err, "unable to parse operator response").Error())
+			fmt.Println(clusterConfig.UserStr())
+			fmt.Println("\n" + errors.Message(err, "unable to parse operator response"))
 			return
 		}
-		infoResponse.ClusterConfig.Config = *clusterConfig
+		infoResponse.ClusterConfig.Config = clusterConfig
 
 		var items table.KeyValuePairs
 		items.Add("aws access key id", infoResponse.MaskedAWSAccessKeyID)
-		items.AddAll(infoResponse.ClusterConfig.UserFacingTable())
+		items.AddAll(infoResponse.ClusterConfig.UserTable())
 
 		items.Print()
 	},
 }
 
-var downCmd = &cobra.Command{
+var _downCmd = &cobra.Command{
 	Use:   "down",
 	Short: "spin down a cluster",
 	Args:  cobra.NoArgs,
@@ -218,7 +218,7 @@ var downCmd = &cobra.Command{
 			exit.Error(err)
 		}
 
-		awsCreds, err := getAWSCredentials(flagClusterConfig)
+		awsCreds, err := getAWSCredentials(_flagClusterConfig)
 		if err != nil {
 			exit.Error(err)
 		}
@@ -243,7 +243,7 @@ var downCmd = &cobra.Command{
 	},
 }
 
-var emailPrompValidation = &cr.PromptValidation{
+var _emailPrompValidation = &cr.PromptValidation{
 	PromptItemValidations: []*cr.PromptItemValidation{
 		{
 			StructField: "EmailAddress",
@@ -266,7 +266,7 @@ func promptForEmail() {
 	emailAddressContainer := &struct {
 		EmailAddress *string
 	}{}
-	err := cr.ReadPrompt(emailAddressContainer, emailPrompValidation)
+	err := cr.ReadPrompt(emailAddressContainer, _emailPrompValidation)
 	if err != nil {
 		exit.Error(err)
 	}
@@ -286,7 +286,7 @@ func promptForEmail() {
 	}
 }
 
-func refreshCachedClusterConfig(awsCreds *AWSCredentials) *clusterconfig.Config {
+func refreshCachedClusterConfig(awsCreds AWSCredentials) clusterconfig.Config {
 	accessConfig, err := getClusterAccessConfig()
 	if err != nil {
 		exit.Error(err)
@@ -313,5 +313,5 @@ func refreshCachedClusterConfig(awsCreds *AWSCredentials) *clusterconfig.Config
 
 	refreshedClusterConfig := &clusterconfig.Config{}
 	readCachedClusterConfigFile(refreshedClusterConfig, cachedConfigPath)
-	return refreshedClusterConfig
+	return *refreshedClusterConfig
 }
diff --git a/cli/cmd/completion.go b/cli/cmd/completion.go
@@ -23,7 +23,7 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var completionCmd = &cobra.Command{
+var _completionCmd = &cobra.Command{
 	Use:   "completion",
 	Short: "generate bash completion scripts",
 	Long: `generate bash completion scripts
@@ -38,7 +38,7 @@ this will also add the "cx" alias (note: cli completion requires the bash_comple
 `,
 	Args: cobra.NoArgs,
 	Run: func(cmd *cobra.Command, args []string) {
-		rootCmd.GenBashCompletion(os.Stdout)
+		_rootCmd.GenBashCompletion(os.Stdout)
 		aliasText := `
 # alias
 

diff --git a/cli/cmd/configure.go b/cli/cmd/configure.go
@@ -19,46 +19,45 @@ package cmd
 import (
 	"fmt"
 
-	"github.com/spf13/cobra"
-
 	"github.com/cortexlabs/cortex/pkg/lib/exit"
 	s "github.com/cortexlabs/cortex/pkg/lib/strings"
 	"github.com/cortexlabs/cortex/pkg/lib/table"
 	"github.com/cortexlabs/cortex/pkg/lib/telemetry"
+	"github.com/spf13/cobra"
 )
 
-var flagPrint bool
+var _flagPrint bool
 
 func init() {
-	addEnvFlag(configureCmd)
-	configureCmd.PersistentFlags().BoolVarP(&flagPrint, "print", "p", false, "print the configuration")
+	addEnvFlag(_configureCmd)
+	_configureCmd.PersistentFlags().BoolVarP(&_flagPrint, "print", "p", false, "print the configuration")
 }
 
-var configureCmd = &cobra.Command{
+var _configureCmd = &cobra.Command{
 	Use:   "configure",
 	Short: "configure the cli",
 	Args:  cobra.NoArgs,
 	Run: func(cmd *cobra.Command, args []string) {
 		telemetry.Event("cli.configure")
 
-		if flagPrint {
-			cliEnvConfig, err := readCLIEnvConfig(flagEnv)
+		if _flagPrint {
+			cliEnvConfig, err := readCLIEnvConfig(_flagEnv)
 			if err != nil {
 				exit.Error(err)
 			}
 
 			if cliEnvConfig == nil {
-				if flagEnv == "default" {
+				if _flagEnv == "default" {
 					exit.Error("cli is not configured; run `cortex configure`")
 				} else {
-					exit.Error(fmt.Sprintf("cli is not configured; run `cortex configure --env=%s`", flagEnv))
+					exit.Error(fmt.Sprintf("cli is not configured; run `cortex configure --env=%s`", _flagEnv))
 				}
 			}
 
 			var items table.KeyValuePairs
 
-			if flagEnv != "default" {
-				items.Add("environment", flagEnv)
+			if _flagEnv != "default" {
+				items.Add("environment", _flagEnv)
 			}
 			items.Add("cortex operator endpoint", cliEnvConfig.OperatorEndpoint)
 			items.Add("aws access key id", cliEnvConfig.AWSAccessKeyID)
@@ -68,6 +67,6 @@ var configureCmd = &cobra.Command{
 			return
 		}
 
-		configureCLIEnv(flagEnv)
+		configureCLIEnv(_flagEnv)
 	},
 }