refactor: split config

.gitignore

@@ -19,7 +19,8 @@ jobs
 /development.env
 /resources/resource_fs.go
 __pycache__
-.optimus.yaml
+optimus.yaml
+config.yaml
 coverage.txt
 
 /api/proto/odpf/**/*

.goreleaser.yml

@@ -10,7 +10,7 @@ builds:
     flags:
       - -a
     ldflags:
-      - -s -w -X github.com/odpf/optimus/config.Version={{.Tag}} -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}}
+      - -s -w -X github.com/odpf/optimus/config.BuildVersion={{.Tag}} -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}}
     goos:
       - linux
       - darwin

Makefile

@@ -14,7 +14,7 @@ PROTON_COMMIT := "f28898cb480769234c5d1e5540b94da1e5949407"
 build: # build optimus binary
 	@echo " > notice: skipped proto generation, use 'generate-proto' make command"
 	@echo " > building optimus version ${OPMS_VERSION}"
-	@go build -ldflags "-X ${NAME}/config.Version=${OPMS_VERSION} -X ${NAME}/config.BuildCommit=${LAST_COMMIT}" -o optimus .
+	@go build -ldflags "-X ${NAME}/config.BuildVersion=${OPMS_VERSION} -X ${NAME}/config.BuildCommit=${LAST_COMMIT}" -o optimus .
 	@echo " - build complete"
 
 test-ci: smoke-test unit-test-ci vet ## run tests

README.md

@@ -97,8 +97,7 @@ Optimus service can be started with
 $ ./optimus serve
 ```
 
-`serve` command has few required configurations that needs to be set for it to start. Configuration can either be stored
-in `.optimus.yaml` file or set as environment variable. Read more about it in [getting started](https://odpf.github.io/optimus/docs/getting-started/configuration).
+`serve` command has few required configurations that needs to be set for it to start. Read more about it in [getting started](https://odpf.github.io/optimus/docs/getting-started/configuration).
 
 ## Compatibility
 

cmd/admin.go

@@ -8,18 +8,38 @@ import (
 )
 
 // adminCommand registers internal administration commands
-func adminCommand(l log.Logger, conf config.Optimus) *cli.Command {
+func adminCommand() *cli.Command {
+	var configFilePath string
+	conf := &config.ClientConfig{}
+	l := initDefaultLogger()
+
 	cmd := &cli.Command{
 		Use:    "admin",
 		Short:  "Internal administration commands",
 		Hidden: true,
 	}
+
+	cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration")
+
+	cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error {
+		// TODO: find a way to load the config in one place
+		var err error
+
+		conf, err = config.LoadClientConfig(configFilePath)
+		if err != nil {
+			return err
+		}
+		l = initClientLogger(conf.Log)
+
+		return nil
+	}
+
 	cmd.AddCommand(adminBuildCommand(l, conf))
 	return cmd
 }
 
 // adminBuildCommand builds a run instance
-func adminBuildCommand(l log.Logger, conf config.Optimus) *cli.Command {
+func adminBuildCommand(l log.Logger, conf *config.ClientConfig) *cli.Command {
 	cmd := &cli.Command{
 		Use:   "build",
 		Short: "Register a job run and get required assets",

cmd/admin_build_instance.go

@@ -27,7 +27,7 @@ const (
 
 const unsubstitutedValue = "<no value>"
 
-func adminBuildInstanceCommand(l log.Logger, conf config.Optimus) *cli.Command {
+func adminBuildInstanceCommand(l log.Logger, conf *config.ClientConfig) *cli.Command {
 	var (
 		optimusHost    = conf.Host
 		projectName    = conf.Project.Name

cmd/backup.go

@@ -4,18 +4,20 @@ import (
 	"time"
 
 	"github.com/MakeNowJust/heredoc"
-	"github.com/odpf/salt/log"
 	cli "github.com/spf13/cobra"
 
 	"github.com/odpf/optimus/config"
-	"github.com/odpf/optimus/models"
 )
 
 const (
 	backupTimeout = time.Minute * 15
 )
 
-func backupCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command {
+func backupCommand() *cli.Command {
+	var configFilePath string
+	conf := &config.ClientConfig{}
+	l := initDefaultLogger()
+
 	cmd := &cli.Command{
 		Use:   "backup",
 		Short: "Backup a resource and its downstream",
@@ -27,8 +29,24 @@ func backupCommand(l log.Logger, conf config.Optimus, datastoreRepo models.Datas
 			"group:core": "true",
 		},
 	}
-	cmd.AddCommand(backupCreateCommand(l, conf, datastoreRepo))
-	cmd.AddCommand(backupListCommand(l, conf, datastoreRepo))
-	cmd.AddCommand(backupStatusCommand(l, conf, datastoreRepo))
+
+	cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration")
+
+	cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error {
+		// TODO: find a way to load the config in one place
+		var err error
+
+		conf, err = config.LoadClientConfig(configFilePath)
+		if err != nil {
+			return err
+		}
+		l = initClientLogger(conf.Log)
+
+		return nil
+	}
+
+	cmd.AddCommand(backupCreateCommand(l, conf))
+	cmd.AddCommand(backupListCommand(l, conf))
+	cmd.AddCommand(backupStatusCommand(l, conf))
 	return cmd
 }

cmd/backup_create.go

@@ -16,7 +16,7 @@ import (
 	"github.com/odpf/optimus/models"
 )
 
-func backupCreateCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command {
+func backupCreateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command {
 	var (
 		backupCmd = &cli.Command{
 			Use:     "create",
@@ -42,11 +42,12 @@ func backupCreateCommand(l log.Logger, conf config.Optimus, datastoreRepo models
 	backupCmd.Flags().BoolVar(&ignoreDownstream, "ignore-downstream", ignoreDownstream, "Do not take backups for dependent downstream resources")
 
 	backupCmd.RunE = func(cmd *cli.Command, args []string) error {
+		var err error
 		namespace, err := askToSelectNamespace(l, conf)
 		if err != nil {
 			return err
 		}
-		if storerName, err = extractDatastoreName(datastoreRepo, storerName); err != nil {
+		if storerName, err = extractDatastoreName(storerName); err != nil {
 			return err
 		}
 		if resourceName, err = extractResourceName(resourceName); err != nil {
@@ -115,9 +116,10 @@ func backupCreateCommand(l log.Logger, conf config.Optimus, datastoreRepo models
 	return backupCmd
 }
 
-func extractDatastoreName(datastoreRepo models.DatastoreRepo, storerName string) (string, error) {
+func extractDatastoreName(storerName string) (string, error) {
+	dsRepo := models.DatastoreRegistry
 	availableStorer := []string{}
-	for _, s := range datastoreRepo.GetAll() {
+	for _, s := range dsRepo.GetAll() {
 		availableStorer = append(availableStorer, s.Name())
 	}
 	if storerName == "" {

cmd/backup_list.go

@@ -16,7 +16,7 @@ import (
 	"github.com/odpf/optimus/models"
 )
 
-func backupListCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command {
+func backupListCommand(l log.Logger, conf *config.ClientConfig) *cli.Command {
 	var (
 		backupCmd = &cli.Command{
 			Use:     "list",
@@ -27,8 +27,9 @@ func backupListCommand(l log.Logger, conf config.Optimus, datastoreRepo models.D
 	)
 	backupCmd.Flags().StringVarP(&project, "project", "p", conf.Project.Name, "project name of optimus managed repository")
 	backupCmd.RunE = func(cmd *cli.Command, args []string) error {
+		dsRepo := models.DatastoreRegistry
 		availableStorer := []string{}
-		for _, s := range datastoreRepo.GetAll() {
+		for _, s := range dsRepo.GetAll() {
 			availableStorer = append(availableStorer, s.Name())
 		}
 		var storerName string

cmd/backup_status.go

@@ -17,7 +17,7 @@ import (
 	"github.com/odpf/optimus/models"
 )
 
-func backupStatusCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command {
+func backupStatusCommand(l log.Logger, conf *config.ClientConfig) *cli.Command {
 	var (
 		project   string
 		backupCmd = &cli.Command{
@@ -28,8 +28,9 @@ func backupStatusCommand(l log.Logger, conf config.Optimus, datastoreRepo models
 	)
 	backupCmd.Flags().StringVarP(&project, "project", "p", conf.Project.Name, "Project name of optimus managed repository")
 	backupCmd.RunE = func(cmd *cli.Command, args []string) error {
+		dsRepo := models.DatastoreRegistry
 		availableStorer := []string{}
-		for _, s := range datastoreRepo.GetAll() {
+		for _, s := range dsRepo.GetAll() {
 			availableStorer = append(availableStorer, s.Name())
 		}
 		var storerName string

cmd/commands.go

@@ -14,7 +14,6 @@ import (
 	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/mattn/go-isatty"
 	"github.com/odpf/salt/cmdx"
-	"github.com/odpf/salt/log"
 	"github.com/odpf/salt/term"
 	"github.com/spf13/afero"
 	cli "github.com/spf13/cobra"
@@ -36,8 +35,7 @@ var (
 		return errors.New(heredoc.Docf(`Unable to reach optimus server at %s, this can happen due to following reasons:
 			1. Check if you are connected to internet
 			2. Is the host correctly configured in optimus config
-			3. Is OPTIMUS_HOST env incorrectly set
-			4. Is Optimus server currently unreachable`, host))
+			3. Is Optimus server currently unreachable`, host))
 	}
 )
 
@@ -62,7 +60,7 @@ type JobSpecRepository interface {
 // default output of logging should go to stdout
 // interactive output like progress bars should go to stderr
 // unless the stdout/err is a tty, colors/progressbar should be disabled
-func New(plainLog, jsonLog log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command {
+func New() *cli.Command {
 	disableColoredOut = !isTerminal(os.Stdout)
 
 	cmd := &cli.Command{
@@ -112,28 +110,19 @@ func New(plainLog, jsonLog log.Logger, conf config.Optimus, pluginRepo models.Pl
 	cmdx.SetHelp(cmd)
 	cmd.PersistentFlags().BoolVar(&disableColoredOut, "no-color", disableColoredOut, "Disable colored output")
 
-	// init local specs
-	datastoreSpecFs := make(map[string]map[string]afero.Fs)
-	for _, namespace := range conf.Namespaces {
-		dtSpec := make(map[string]afero.Fs)
-		for _, dsConfig := range namespace.Datastore {
-			dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path)
-		}
-		datastoreSpecFs[namespace.Name] = dtSpec
-	}
+	cmd.AddCommand(versionCommand())
+	cmd.AddCommand(configCommand())
+	cmd.AddCommand(jobCommand())
+	cmd.AddCommand(deployCommand())
+	cmd.AddCommand(resourceCommand())
+	cmd.AddCommand(replayCommand())
+	cmd.AddCommand(backupCommand())
+	cmd.AddCommand(adminCommand())
+	cmd.AddCommand(secretCommand())
+
+	cmd.AddCommand(serveCommand())
 
-	cmd.AddCommand(versionCommand(plainLog, conf.Host, pluginRepo))
-	cmd.AddCommand(configCommand(plainLog))
-	cmd.AddCommand(jobCommand(plainLog, conf, pluginRepo))
-	cmd.AddCommand(deployCommand(plainLog, conf, pluginRepo, dsRepo, datastoreSpecFs))
-	cmd.AddCommand(resourceCommand(plainLog, conf, dsRepo, datastoreSpecFs))
-	cmd.AddCommand(serveCommand(jsonLog, conf))
-	cmd.AddCommand(replayCommand(plainLog, conf))
-	cmd.AddCommand(backupCommand(plainLog, conf, dsRepo))
-	cmd.AddCommand(adminCommand(plainLog, conf))
-	cmd.AddCommand(secretCommand(plainLog, conf))
-
-	addExtensionCommand(cmd, plainLog)
+	addExtensionCommand(cmd)
 	return cmd
 }
 
@@ -206,3 +195,15 @@ func (a *BasicAuthentication) RequireTransportSecurity() bool {
 func isTerminal(f *os.File) bool {
 	return isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd())
 }
+
+func getDatastoreSpecFs(namespaces []*config.Namespace) map[string]map[string]afero.Fs {
+	output := make(map[string]map[string]afero.Fs)
+	for _, namespace := range namespaces {
+		dtSpec := make(map[string]afero.Fs)
+		for _, dsConfig := range namespace.Datastore {
+			dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path)
+		}
+		output[namespace.Name] = dtSpec
+	}
+	return output
+}

cmd/config.go

@@ -5,33 +5,33 @@ import (
 	"io/ioutil"
 
 	"github.com/AlecAivazis/survey/v2"
-	"github.com/odpf/salt/log"
 	cli "github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
 	"github.com/odpf/optimus/config"
 )
 
 const (
-	defaultHost = "localhost"
+	defaultHost               = "localhost"
+	defaultFilePermissionMode = 0o655
 )
 
-func configCommand(l log.Logger) *cli.Command {
+func configCommand() *cli.Command {
 	c := &cli.Command{
 		Use:   "config",
 		Short: "Manage optimus configuration required to deploy specifications",
 	}
-	c.AddCommand(configInitCommand(l))
+	c.AddCommand(configInitCommand())
 	return c
 }
 
-func configInitCommand(l log.Logger) *cli.Command {
+func configInitCommand() *cli.Command {
 	c := &cli.Command{
 		Use:   "init",
 		Short: "Initialize optimus configuration file",
 		RunE: func(c *cli.Command, args []string) (err error) {
-			conf := config.Optimus{
-				Version: 1,
+			conf := config.ClientConfig{
+				Version: config.Version(1),
 				Host:    defaultHost,
 			}
 			questions := []*survey.Question{
@@ -93,17 +93,20 @@ func configInitCommand(l log.Logger) *cli.Command {
 			if err != nil {
 				return err
 			}
-			if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.FileName, config.FileExtension), confMarshaled, 0o600); err != nil {
+			filePath := fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension)
+			if err := ioutil.WriteFile(filePath, confMarshaled, defaultFilePermissionMode); err != nil {
 				return err
 			}
+
+			l := initClientLogger(conf.Log)
 			l.Info(coloredSuccess("Configuration initialised successfully"))
 			return nil
 		},
 	}
 	return c
 }
 
-func projectConfigQuestions(conf config.Optimus) (config.Optimus, error) {
+func projectConfigQuestions(conf config.ClientConfig) (config.ClientConfig, error) {
 	conf.Project.Config = map[string]string{}
 	registerMore := AnswerYes
 	for registerMore == AnswerNo {