From 12ea3b8dc06fffb270e1d21a9442bb46e96cd70f Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 21 Mar 2022 16:58:23 +0700 Subject: [PATCH 01/54] refactor: add placeholder for config structure --- .optimus.sample.yaml | 78 ++++++++++--------- config/build.go | 6 +- config/config.go | 127 ------------------------------- config/config_common.go | 17 +++++ config/config_project.go | 33 ++++++++ config/config_server.go | 49 ++++++++++++ config/loader.go | 158 ++++++++++++++++++++++++++------------- config/loader_test.go | 150 ++++++++++++++++++------------------- config/telemetry.go | 2 +- 9 files changed, 327 insertions(+), 293 deletions(-) delete mode 100644 config/config.go create mode 100644 config/config_common.go create mode 100644 config/config_project.go create mode 100644 config/config_server.go diff --git a/.optimus.sample.yaml b/.optimus.sample.yaml index 22dcc75797..c0d3e03789 100644 --- a/.optimus.sample.yaml +++ b/.optimus.sample.yaml @@ -1,46 +1,19 @@ version: 1 +######################################## +# COMMON CONFIG (APPLIED FOR SERVER & CLIENT) +######################################## # logging configuration log: # debug, info, warning, error, fatal - default 'info' level: info -# -# cli configurations -# -# used to connect optimus service -#host: localhost:9100 -# for configuring optimus project -#project: -# name: sample_project -# # project variables usable in specifications -# config: -# environment: integration -# scheduler_host: http://example.io/ -# # storage_path is used for storing compiled job specifications that can be -# # consumed by schedulers like Airflow -# # it supports multiple schemes like: file://, gcs:// -# storage_path: file://absolute_path_to_a_directory -# for configuring optimus namespace -#namespace: -# name: sample_namespace -# job: -# # relative path pointing to folder where job specifications are stored -# path: "job" -# datastore: -# # optimus is capable of supporting multiple datastores -# type: bigquery -# # relative path where resource spec for BQ are stored -# path: "bq" -# # namespace variables usable in specifications -# config: {} - -# -# server configurations -# +######################################## +# SERVER CONFIG +######################################## # for configuring optimus service #serve: @@ -83,4 +56,41 @@ log: # profile_addr: ":9110" # # # jaeger collector address to send application traces -# jaeger_addr: "http://localhost:14268/api/traces" \ No newline at end of file +# jaeger_addr: "http://localhost:14268/api/traces" + + + + +######################################## +# PROJECT CONFIG +######################################## + +# used to connect optimus service +#host: localhost:9100 + +# for configuring optimus project +#project: +# name: sample_project +# # project variables usable in specifications +# config: +# environment: integration +# scheduler_host: http://example.io/ +# # storage_path is used for storing compiled job specifications that can be +# # consumed by schedulers like Airflow +# # it supports multiple schemes like: file://, gcs:// +# storage_path: file://absolute_path_to_a_directory + +# for configuring optimus namespace +#namespace: +# name: sample_namespace +# job: +# # relative path pointing to folder where job specifications are stored +# path: "job" +# datastore: +# # optimus is capable of supporting multiple datastores +# type: bigquery +# # relative path where resource spec for BQ are stored +# path: "bq" +# # namespace variables usable in specifications +# config: {} + diff --git a/config/build.go b/config/build.go index f1ca14fb93..6109b472e8 100644 --- a/config/build.go +++ b/config/build.go @@ -9,9 +9,9 @@ const ( var ( // overridden by the build system - Version = "dev" - BuildCommit = "" - BuildDate = "" + BuildVersion = "dev" + BuildCommit = "" + BuildDate = "" ) // AppName returns the name used as identifier in telemetry diff --git a/config/config.go b/config/config.go deleted file mode 100644 index b29a0cb794..0000000000 --- a/config/config.go +++ /dev/null @@ -1,127 +0,0 @@ -package config - -import ( - "fmt" - "strconv" - "time" -) - -const ( - KeyServeReplayNumWorkers = "serve.replay_num_workers" -) - -type Optimus struct { - // configuration version - Version int `mapstructure:"version"` - // optimus server host - Host string `mapstructure:"host"` - - Project Project `mapstructure:"project"` - Namespaces []*Namespace `mapstructure:"namespaces"` - - Server ServerConfig `mapstructure:"serve"` - Log LogConfig `mapstructure:"log"` - Scheduler SchedulerConfig `mapstructure:"scheduler"` - Telemetry TelemetryConfig `mapstructure:"telemetry"` - - namespaceNameToNamespace map[string]*Namespace -} - -func (o *Optimus) GetNamespaceByName(name string) (*Namespace, error) { - if o.namespaceNameToNamespace == nil { - o.namespaceNameToNamespace = make(map[string]*Namespace) - for _, namespace := range o.Namespaces { - o.namespaceNameToNamespace[namespace.Name] = namespace - } - } - if o.namespaceNameToNamespace[name] == nil { - return nil, fmt.Errorf("namespace [%s] is not found", name) - } - return o.namespaceNameToNamespace[name], nil -} - -type Datastore struct { - // type could be bigquery/postgres/gcs - Type string `mapstructure:"type"` - - // directory to find specifications - Path string `mapstructure:"path"` - - // backup configuration - Backup map[string]string `mapstructure:"backup"` -} - -type Job struct { - // directory to find specifications relative to where the config is located - Path string `mapstructure:"path"` -} - -type Project struct { - Name string `mapstructure:"name"` - Config map[string]string `mapstructure:"config"` -} - -type Namespace struct { - Name string `mapstructure:"name"` - Config map[string]string `mapstructure:"config"` - Job Job `mapstructure:"job"` - Datastore []Datastore `mapstructure:"datastore"` -} - -type LogConfig struct { - // log level - debug, info, warning, error, fatal - Level string `mapstructure:"level" default:"info"` - - // format strategy - plain, json - Format string `mapstructure:"format"` -} - -type ServerConfig struct { - // port to listen on - Port int `mapstructure:"port" default:"9100"` - // the network interface to listen on - Host string `mapstructure:"host" default:"0.0.0.0"` - - // service ingress host for jobs to communicate back to optimus - IngressHost string `mapstructure:"ingress_host"` - - // random 32 character hash used for encrypting secrets - AppKey string `mapstructure:"app_key"` - - DB DBConfig `mapstructure:"db"` - ReplayNumWorkers int `mapstructure:"replay_num_workers" default:"1"` - ReplayWorkerTimeout time.Duration `mapstructure:"replay_worker_timeout" default:"120s"` - ReplayRunTimeout time.Duration `mapstructure:"replay_run_timeout"` -} - -type DBConfig struct { - // database connection string - // e.g.: postgres://user:password@host:123/database?sslmode=disable - DSN string `mapstructure:"dsn"` - - // maximum allowed idle DB connections - MaxIdleConnection int `mapstructure:"max_idle_connection" default:"10"` - - // maximum allowed open DB connections - MaxOpenConnection int `mapstructure:"max_open_connection" default:"20"` -} - -type SchedulerConfig struct { - Name string `mapstructure:"name" default:"airflow2"` - SkipInit bool `mapstructure:"skip_init"` - - RaftAddr string `mapstructure:"raft_addr"` - GossipAddr string `mapstructure:"gossip_addr"` - NodeID string `mapstructure:"node_id"` - DataDir string `mapstructure:"data_dir"` - Peers string `mapstructure:"peers"` -} - -type TelemetryConfig struct { - ProfileAddr string `mapstructure:"profile_addr"` - JaegerAddr string `mapstructure:"jaeger_addr"` -} - -func (o *Optimus) GetVersion() string { - return strconv.Itoa(o.Version) -} diff --git a/config/config_common.go b/config/config_common.go new file mode 100644 index 0000000000..c31724fa66 --- /dev/null +++ b/config/config_common.go @@ -0,0 +1,17 @@ +package config + +import "strconv" + +// Contains shared config for server and client (project) + +// Version implement fmt.Stringer +type Version int + +type LogConfig struct { + Level string `mapstructure:"level" default:"info"` // log level - debug, info, warning, error, fatal + Format string `mapstructure:"format"` // format strategy - plain, json +} + +func (v Version) String() string { + return strconv.Itoa(int(v)) +} diff --git a/config/config_project.go b/config/config_project.go new file mode 100644 index 0000000000..0442b803fd --- /dev/null +++ b/config/config_project.go @@ -0,0 +1,33 @@ +package config + +type ProjectConfig struct { + Version int `mapstructure:"version"` + Log LogConfig `mapstructure:"log"` + Host string `mapstructure:"host"` // optimus server host + Project Project `mapstructure:"project"` + Namespaces []*Namespace `mapstructure:"namespaces"` + + namespaceNameToNamespace map[string]*Namespace +} + +type Datastore struct { + Type string `mapstructure:"type"` // type could be bigquery/postgres/gcs + Path string `mapstructure:"path"` // directory to find specifications + Backup map[string]string `mapstructure:"backup"` // backup configuration +} + +type Job struct { + Path string `mapstructure:"path"` // directory to find specifications +} + +type Project struct { + Name string `mapstructure:"name"` + Config map[string]string `mapstructure:"config"` +} + +type Namespace struct { + Name string `mapstructure:"name"` + Config map[string]string `mapstructure:"config"` + Job Job `mapstructure:"job"` + Datastore []Datastore `mapstructure:"datastore"` +} diff --git a/config/config_server.go b/config/config_server.go new file mode 100644 index 0000000000..e3940cae35 --- /dev/null +++ b/config/config_server.go @@ -0,0 +1,49 @@ +package config + +import ( + "time" +) + +const ( + KeyServeReplayNumWorkers = "serve.replay_num_workers" +) + +type ServerConfig struct { + Version Version `mapstructure:"version"` + Log LogConfig `mapstructure:"log"` + Serve Serve `mapstructure:"serve"` + Scheduler SchedulerConfig `mapstructure:"scheduler"` + Telemetry TelemetryConfig `mapstructure:"telemetry"` +} + +type Serve struct { + Port int `mapstructure:"port" default:"9100"` // port to listen on + Host string `mapstructure:"host" default:"0.0.0.0"` // the network interface to listen on + IngressHost string `mapstructure:"ingress_host"` // service ingress host for jobs to communicate back to optimus + AppKey string `mapstructure:"app_key"` // random 32 character hash used for encrypting secrets + DB DBConfig `mapstructure:"db"` + ReplayNumWorkers int `mapstructure:"replay_num_workers" default:"1"` + ReplayWorkerTimeout time.Duration `mapstructure:"replay_worker_timeout" default:"120s"` + ReplayRunTimeout time.Duration `mapstructure:"replay_run_timeout"` +} + +type DBConfig struct { + DSN string `mapstructure:"dsn"` // data source name e.g.: postgres://user:password@host:123/database?sslmode=disable + MaxIdleConnection int `mapstructure:"max_idle_connection" default:"10"` // maximum allowed idle DB connections + MaxOpenConnection int `mapstructure:"max_open_connection" default:"20"` // maximum allowed open DB connections +} + +type SchedulerConfig struct { + Name string `mapstructure:"name" default:"airflow2"` + SkipInit bool `mapstructure:"skip_init"` + RaftAddr string `mapstructure:"raft_addr"` + GossipAddr string `mapstructure:"gossip_addr"` + NodeID string `mapstructure:"node_id"` + DataDir string `mapstructure:"data_dir"` + Peers string `mapstructure:"peers"` +} + +type TelemetryConfig struct { + ProfileAddr string `mapstructure:"profile_addr"` + JaegerAddr string `mapstructure:"jaeger_addr"` +} diff --git a/config/loader.go b/config/loader.go index 83f836337b..09eb0f7a36 100644 --- a/config/loader.go +++ b/config/loader.go @@ -2,8 +2,6 @@ package config import ( "errors" - "fmt" - "os" "strings" "github.com/odpf/salt/config" @@ -12,60 +10,69 @@ import ( ) const ( - ErrFailedToRead = "unable to read optimus config file %v (%s)" - FileName = ".optimus" - FileExtension = "yaml" + ErrFailedToRead = "unable to read optimus config file %v (%s)" + DefaultFilename = ".optimus" + DefaultFileExtension = "yaml" ) -type LoadConfigFunc func(interface{}, afero.Fs, ...string) error - -// LoadOptimusConfig Load configuration file from following paths -// ./ -// / -// ~/.optimus/ -// Namespaces will be loaded only from current project ./ -func LoadOptimusConfig(dirPaths ...string) (*Optimus, error) { - fs := afero.NewReadOnlyFs(afero.NewOsFs()) - - var targetPath string - if len(dirPaths) > 0 { - targetPath = dirPaths[0] - } else { - currPath, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("error getting current work directory path: %w", err) - } - targetPath = currPath - } +var ( + filename = DefaultFilename + fileExtension = DefaultFileExtension // ASK: are we providing file extension other than yaml? +) - optimus := Optimus{} - if err := loadConfig(&optimus, fs, targetPath); err != nil { - return nil, errors.New("error loading config") - } - if err := validateNamespaceDuplication(&optimus); err != nil { - return nil, err - } - return &optimus, nil +// MustLoadProjectConfig load the project specific config (see LoadProjectConfig) with panic +func MustLoadProjectConfig(path ...string) *ProjectConfig { + // implement this + return nil } -func validateNamespaceDuplication(optimus *Optimus) error { - nameToAppearance := make(map[string]int) - for _, namespace := range optimus.Namespaces { - nameToAppearance[namespace.Name]++ - } - var duplicateNames []string - for name, appearance := range nameToAppearance { - if appearance > 1 { - duplicateNames = append(duplicateNames, name) - } - } - if len(duplicateNames) > 0 { - return fmt.Errorf("namespaces [%s] are duplicate", strings.Join(duplicateNames, ", ")) +// LoadProjectConfig load the project specific config from these locations: +// 1. env var. eg. OPTIMUS_PROJECT, OPTIMUS_NAMESPACES, etc +// 2. path. ./optimus -c "path/to/config/optimus.yaml" +// 3. current dir. Optimus will look at current directory if there's optimus.yaml there, use it +func LoadProjectConfig(path ...string) (*ProjectConfig, error) { + // implement this + return nil, nil +} + +// MustLoadServerConfig load the server specific config (see LoadServerConfig) with panic +func MustLoadServerConfig(path ...string) *ServerConfig { + // implement this + return nil +} + +// LoadServerConfig load the server specific config from these locations: +// 1. env var. eg. OPTIMUS_SERVE_PORT, etc +// 2. path. ./optimus -c "path/to/config.yaml" +// 3. executable binary location +// 4. home dir +func LoadServerConfig(path ...string) (*ServerConfig, error) { + // implement this + return nil, nil +} + +// Validate validate the config as an input. If not valid, it returns error +func Validate(conf interface{}) error { + switch c := conf.(type) { + case ProjectConfig: + return validateProjectConfig(c) + case ServerConfig: + return validateServerConfig(c) } + return errors.New("error") +} + +func validateProjectConfig(conf ProjectConfig) error { + // implement this return nil } -func loadConfig(cfg interface{}, fs afero.Fs, dirPath string) error { +func validateServerConfig(conf ServerConfig) error { + // implement this + return nil +} + +func loadConfig(cfg interface{}, fs afero.Fs, dirPaths ...string) error { // getViperWithDefault + SetFs v := viper.New() v.SetConfigName("config") @@ -75,13 +82,64 @@ func loadConfig(cfg interface{}, fs afero.Fs, dirPath string) error { opts := []config.LoaderOption{ config.WithViper(v), - config.WithName(FileName), - config.WithType(FileExtension), + config.WithName(filename), + config.WithType(fileExtension), config.WithEnvPrefix("OPTIMUS"), config.WithEnvKeyReplacer(".", "_"), - config.WithPath(dirPath), + } + + for _, path := range dirPaths { + opts = append(opts, config.WithPath(path)) } l := config.NewLoader(opts...) return l.Load(cfg) } + +// type LoadConfigFunc func(interface{}, afero.Fs, ...string) error + +// // LoadOptimusConfig Load configuration file from following paths +// // ./ +// // / +// // ~/.optimus/ +// // Namespaces will be loaded only from current project ./ +// func LoadOptimusConfig(dirPaths ...string) (*Optimus, error) { +// fs := afero.NewReadOnlyFs(afero.NewOsFs()) + +// var targetPath string +// if len(dirPaths) > 0 { +// targetPath = dirPaths[0] +// } else { +// currPath, err := os.Getwd() +// if err != nil { +// return nil, fmt.Errorf("error getting current work directory path: %w", err) +// } +// targetPath = currPath +// } + +// optimus := Optimus{} +// if err := loadConfig(&optimus, fs, targetPath); err != nil { +// return nil, errors.New("error loading config") +// } +// if err := validateNamespaceDuplication(&optimus); err != nil { +// return nil, err +// } +// return &optimus, nil +// } + +// func validateNamespaceDuplication(optimus *Optimus) error { +// nameToAppearance := make(map[string]int) +// for _, namespace := range optimus.Namespaces { +// nameToAppearance[namespace.Name]++ +// } +// var duplicateNames []string +// for name, appearance := range nameToAppearance { +// if appearance > 1 { +// duplicateNames = append(duplicateNames, name) +// } +// } +// if len(duplicateNames) > 0 { +// return fmt.Errorf("namespaces [%s] are duplicate", strings.Join(duplicateNames, ", ")) +// } +// return nil +// } diff --git a/config/loader_test.go b/config/loader_test.go index 668868cc8f..38185439d4 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -3,12 +3,6 @@ package config_test import ( "os" "path" - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/odpf/optimus/config" ) const ( @@ -71,82 +65,82 @@ func teardown() { } } -func TestLoadOptimusConfig(t *testing.T) { - t.Run("should return config and nil if no error is found", func(t *testing.T) { - setup(optimusConfigContent + ` -- name: namespace-b - job: - path: ./jobs-b -`) - defer teardown() +// func TestLoadOptimusConfig(t *testing.T) { +// t.Run("should return config and nil if no error is found", func(t *testing.T) { +// setup(optimusConfigContent + ` +// - name: namespace-b +// job: +// path: ./jobs-b +// `) +// defer teardown() - expectedErrMsg := "namespaces [namespace-b] are duplicate" +// expectedErrMsg := "namespaces [namespace-b] are duplicate" - actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) +// actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) - assert.Nil(t, actualConf) - assert.EqualError(t, actualErr, expectedErrMsg) - }) +// assert.Nil(t, actualConf) +// assert.EqualError(t, actualErr, expectedErrMsg) +// }) - t.Run("should return config and nil if no error is found", func(t *testing.T) { - setup(optimusConfigContent) - defer teardown() +// t.Run("should return config and nil if no error is found", func(t *testing.T) { +// setup(optimusConfigContent) +// defer teardown() - expectedConf := &config.Optimus{ - Version: 1, - Log: config.LogConfig{ - Level: "info", - }, - Host: "localhost:9100", - Project: config.Project{ - Name: "sample_project", - Config: map[string]string{ - "environment": "integration", - "scheduler_host": "http://example.io/", - "storage_path": "file://absolute_path_to_a_directory", - }, - }, - Server: config.ServerConfig{ - Port: 9100, - Host: "localhost", - IngressHost: "optimus.example.io:80", - AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", - ReplayNumWorkers: 1, - ReplayWorkerTimeout: 100 * time.Second, - ReplayRunTimeout: 10 * time.Second, - DB: config.DBConfig{ - DSN: "postgres://user:password@localhost:5432/database?sslmode=disable", - MaxIdleConnection: 5, - MaxOpenConnection: 10, - }, - }, - Scheduler: config.SchedulerConfig{ - Name: "airflow2", - SkipInit: true, - }, - Telemetry: config.TelemetryConfig{ - ProfileAddr: ":9110", - JaegerAddr: "http://localhost:14268/api/traces", - }, - Namespaces: []*config.Namespace{ - { - Name: "namespace-a", - Job: config.Job{ - Path: "./jobs-a", - }, - }, - { - Name: "namespace-b", - Job: config.Job{ - Path: "./jobs-b", - }, - }, - }, - } +// expectedConf := &config.Optimus{ +// Version: 1, +// Log: config.LogConfig{ +// Level: "info", +// }, +// Host: "localhost:9100", +// Project: config.Project{ +// Name: "sample_project", +// Config: map[string]string{ +// "environment": "integration", +// "scheduler_host": "http://example.io/", +// "storage_path": "file://absolute_path_to_a_directory", +// }, +// }, +// Server: config.ServerConfig{ +// Port: 9100, +// Host: "localhost", +// IngressHost: "optimus.example.io:80", +// AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", +// ReplayNumWorkers: 1, +// ReplayWorkerTimeout: 100 * time.Second, +// ReplayRunTimeout: 10 * time.Second, +// DB: config.DBConfig{ +// DSN: "postgres://user:password@localhost:5432/database?sslmode=disable", +// MaxIdleConnection: 5, +// MaxOpenConnection: 10, +// }, +// }, +// Scheduler: config.SchedulerConfig{ +// Name: "airflow2", +// SkipInit: true, +// }, +// Telemetry: config.TelemetryConfig{ +// ProfileAddr: ":9110", +// JaegerAddr: "http://localhost:14268/api/traces", +// }, +// Namespaces: []*config.Namespace{ +// { +// Name: "namespace-a", +// Job: config.Job{ +// Path: "./jobs-a", +// }, +// }, +// { +// Name: "namespace-b", +// Job: config.Job{ +// Path: "./jobs-b", +// }, +// }, +// }, +// } - actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) +// actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) - assert.EqualValues(t, expectedConf, actualConf) - assert.NoError(t, actualErr) - }) -} +// assert.EqualValues(t, expectedConf, actualConf) +// assert.NoError(t, actualErr) +// }) +// } diff --git a/config/telemetry.go b/config/telemetry.go index 369e16d898..e23b7b872f 100644 --- a/config/telemetry.go +++ b/config/telemetry.go @@ -106,7 +106,7 @@ func tracerProvider(url string) (*tracesdk.TracerProvider, error) { tracesdk.WithResource(resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNameKey.String(AppName()), - semconv.ServiceVersionKey.String(Version), + semconv.ServiceVersionKey.String(BuildVersion), attribute.String("build_commit", BuildCommit), attribute.String("build_date", BuildDate), )), From 0651891292f227785134f3898b244d07858aa25d Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 22 Mar 2022 09:14:44 +0700 Subject: [PATCH 02/54] refactor: version command to not accept conf directly --- cmd/commands.go | 8 ++--- cmd/version.go | 79 +++++++++++++++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/cmd/commands.go b/cmd/commands.go index 56b7d0cc9d..21761e2944 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -13,6 +13,7 @@ import ( grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/mattn/go-isatty" + "github.com/odpf/optimus/models" "github.com/odpf/salt/cmdx" "github.com/odpf/salt/log" "github.com/odpf/salt/term" @@ -20,9 +21,6 @@ import ( cli "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" - - "github.com/odpf/optimus/config" - "github.com/odpf/optimus/models" ) var ( @@ -62,7 +60,7 @@ type JobSpecRepository interface { // default output of logging should go to stdout // interactive output like progress bars should go to stderr // unless the stdout/err is a tty, colors/progressbar should be disabled -func New(plainLog, jsonLog log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { +func New(plainLog log.Logger, jsonLog log.Logger, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { disableColoredOut = !isTerminal(os.Stdout) cmd := &cli.Command{ @@ -122,7 +120,7 @@ func New(plainLog, jsonLog log.Logger, conf config.Optimus, pluginRepo models.Pl datastoreSpecFs[namespace.Name] = dtSpec } - cmd.AddCommand(versionCommand(plainLog, conf.Host, pluginRepo)) + cmd.AddCommand(versionCommand(plainLog, pluginRepo)) cmd.AddCommand(configCommand(plainLog)) cmd.AddCommand(jobCommand(plainLog, conf, pluginRepo)) cmd.AddCommand(deployCommand(plainLog, conf, pluginRepo, dsRepo, datastoreSpecFs)) diff --git a/cmd/version.go b/cmd/version.go index a57d8ebd50..81bff5a813 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -20,47 +20,62 @@ const ( githubRepo = "odpf/optimus" ) -func versionCommand(l log.Logger, host string, pluginRepo models.PluginRepository) *cli.Command { - var serverVersion bool +func versionCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command { + var isWithServer bool + c := &cli.Command{ Use: "version", Short: "Print the client version information", Example: "optimus version [--with-server]", - RunE: func(c *cli.Command, args []string) error { - l.Info(fmt.Sprintf("Client: %s-%s", coloredNotice(config.Version), - coloredNotice(config.BuildCommit))) - if host != "" && serverVersion { - srvVer, err := getVersionRequest(config.Version, host) - if err != nil { - return err - } - l.Info(fmt.Sprintf("Server: %s", coloredNotice(srvVer))) + } + + c.Flags().BoolVar(&isWithServer, "with-server", false, "Check for server version") + + c.RunE = func(c *cli.Command, args []string) error { + // Print client version + l.Info(fmt.Sprintf("Client: %s-%s", coloredNotice(config.BuildVersion), coloredNotice(config.BuildCommit))) + + // Print server version + if isWithServer { + conf, err := config.LoadProjectConfig() + if err != nil { + return err } - if updateNotice := version.UpdateNotice(config.Version, githubRepo); updateNotice != "" { - l.Info(updateNotice) + + srvVer, err := getVersionRequest(config.BuildVersion, conf.Host) + if err != nil { + return err } - plugins := pluginRepo.GetAll() - l.Info(fmt.Sprintf("\nDiscovered plugins: %d", len(plugins))) - for taskIdx, tasks := range plugins { - schema := tasks.Info() - l.Info(fmt.Sprintf("\n%d. %s", taskIdx+1, schema.Name)) - l.Info(fmt.Sprintf("Description: %s", schema.Description)) - l.Info(fmt.Sprintf("Image: %s", schema.Image)) - l.Info(fmt.Sprintf("Type: %s", schema.PluginType)) - l.Info(fmt.Sprintf("Plugin version: %s", schema.PluginVersion)) - l.Info(fmt.Sprintf("Plugin mods: %v", schema.PluginMods)) - if schema.HookType != "" { - l.Info(fmt.Sprintf("Hook type: %s", schema.HookType)) - } - if len(schema.DependsOn) != 0 { - l.Info(fmt.Sprintf("Depends on: %v", schema.DependsOn)) - } + l.Info(fmt.Sprintf("Server: %s", coloredNotice(srvVer))) + } + + // Print version update if new version is exist + if updateNotice := version.UpdateNotice(config.BuildVersion, githubRepo); updateNotice != "" { + l.Info(updateNotice) + } + + // Print all plugin infos + plugins := pluginRepo.GetAll() + l.Info(fmt.Sprintf("\nDiscovered plugins: %d", len(plugins))) + for taskIdx, tasks := range plugins { + schema := tasks.Info() + l.Info(fmt.Sprintf("\n%d. %s", taskIdx+1, schema.Name)) + l.Info(fmt.Sprintf("Description: %s", schema.Description)) + l.Info(fmt.Sprintf("Image: %s", schema.Image)) + l.Info(fmt.Sprintf("Type: %s", schema.PluginType)) + l.Info(fmt.Sprintf("Plugin version: %s", schema.PluginVersion)) + l.Info(fmt.Sprintf("Plugin mods: %v", schema.PluginMods)) + if schema.HookType != "" { + l.Info(fmt.Sprintf("Hook type: %s", schema.HookType)) } - return nil - }, + if len(schema.DependsOn) != 0 { + l.Info(fmt.Sprintf("Depends on: %v", schema.DependsOn)) + } + } + return nil } - c.Flags().BoolVar(&serverVersion, "with-server", false, "Check for server version") + return c } From de6bcfe04036fbb0c737d65fe39602e355d30850 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 22 Mar 2022 10:39:17 +0700 Subject: [PATCH 03/54] feat: implement load project and server config --- config/config_project.go | 2 +- config/loader.go | 94 ++++++++++++++++++++++++++++++---------- 2 files changed, 73 insertions(+), 23 deletions(-) diff --git a/config/config_project.go b/config/config_project.go index 0442b803fd..4261d2caef 100644 --- a/config/config_project.go +++ b/config/config_project.go @@ -1,7 +1,7 @@ package config type ProjectConfig struct { - Version int `mapstructure:"version"` + Version Version `mapstructure:"version"` Log LogConfig `mapstructure:"log"` Host string `mapstructure:"host"` // optimus server host Project Project `mapstructure:"project"` diff --git a/config/loader.go b/config/loader.go index 09eb0f7a36..e0193d2a3a 100644 --- a/config/loader.go +++ b/config/loader.go @@ -2,7 +2,7 @@ package config import ( "errors" - "strings" + "os" "github.com/odpf/salt/config" "github.com/spf13/afero" @@ -13,42 +13,89 @@ const ( ErrFailedToRead = "unable to read optimus config file %v (%s)" DefaultFilename = ".optimus" DefaultFileExtension = "yaml" + DefaultEnvPrefix = "OPTIMUS" ) var ( filename = DefaultFilename fileExtension = DefaultFileExtension // ASK: are we providing file extension other than yaml? + envPrefix = DefaultEnvPrefix + currPath string + execPath string + homePath string ) +func init() { + p, err := os.Getwd() + if err != nil { + panic(err) + } + currPath = p + + p, err = os.Executable() + if err != nil { + panic(err) + } + execPath = p + + p, err = os.UserHomeDir() + if err != nil { + panic(err) + } + homePath = p +} + // MustLoadProjectConfig load the project specific config (see LoadProjectConfig) with panic -func MustLoadProjectConfig(path ...string) *ProjectConfig { - // implement this - return nil +func MustLoadProjectConfig(filepath ...string) *ProjectConfig { + cfg, err := LoadProjectConfig(filepath...) + if err != nil { + panic(err) + } + + return cfg } // LoadProjectConfig load the project specific config from these locations: // 1. env var. eg. OPTIMUS_PROJECT, OPTIMUS_NAMESPACES, etc -// 2. path. ./optimus -c "path/to/config/optimus.yaml" +// 2. filepath. ./optimus -c "path/to/config/optimus.yaml" // 3. current dir. Optimus will look at current directory if there's optimus.yaml there, use it -func LoadProjectConfig(path ...string) (*ProjectConfig, error) { - // implement this - return nil, nil +func LoadProjectConfig(filepath ...string) (*ProjectConfig, error) { + cfg := &ProjectConfig{} + fs := afero.NewReadOnlyFs(afero.NewOsFs()) + + err := loadConfig(cfg, fs, filepath[0], currPath) + if err != nil { + return nil, err + } + + return cfg, nil } // MustLoadServerConfig load the server specific config (see LoadServerConfig) with panic -func MustLoadServerConfig(path ...string) *ServerConfig { - // implement this - return nil +func MustLoadServerConfig(filepath ...string) *ServerConfig { + cfg, err := LoadServerConfig(filepath...) + if err != nil { + panic(err) + } + + return cfg } // LoadServerConfig load the server specific config from these locations: // 1. env var. eg. OPTIMUS_SERVE_PORT, etc -// 2. path. ./optimus -c "path/to/config.yaml" +// 2. filepath. ./optimus -c "path/to/config.yaml" // 3. executable binary location // 4. home dir -func LoadServerConfig(path ...string) (*ServerConfig, error) { - // implement this - return nil, nil +func LoadServerConfig(filepath ...string) (*ServerConfig, error) { + cfg := &ServerConfig{} + fs := afero.NewReadOnlyFs(afero.NewOsFs()) + + err := loadConfig(cfg, fs, filepath[0], execPath, homePath) + if err != nil { + return nil, err + } + + return cfg, nil } // Validate validate the config as an input. If not valid, it returns error @@ -72,24 +119,27 @@ func validateServerConfig(conf ServerConfig) error { return nil } -func loadConfig(cfg interface{}, fs afero.Fs, dirPaths ...string) error { +func loadConfig(cfg interface{}, fs afero.Fs, paths ...string) error { // getViperWithDefault + SetFs v := viper.New() - v.SetConfigName("config") - v.SetConfigType("yaml") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.SetFs(fs) opts := []config.LoaderOption{ config.WithViper(v), config.WithName(filename), config.WithType(fileExtension), - config.WithEnvPrefix("OPTIMUS"), + config.WithEnvPrefix(envPrefix), config.WithEnvKeyReplacer(".", "_"), } - for _, path := range dirPaths { - opts = append(opts, config.WithPath(path)) + if len(paths) > 0 { + filepath := paths[0] + dirPaths := paths[1:] + + opts = append(opts, config.WithFile(filepath)) + for _, path := range dirPaths { + opts = append(opts, config.WithPath(path)) + } } l := config.NewLoader(opts...) From b22d9579dfd99b480e07066bb202439d77306165 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 22 Mar 2022 18:33:06 +0700 Subject: [PATCH 04/54] refactor: load project config to accept filepath --- config/loader.go | 41 +++++++++- config/loader_test.go | 180 ++++++++++++++++++++++++++++++++++++------ 2 files changed, 192 insertions(+), 29 deletions(-) diff --git a/config/loader.go b/config/loader.go index e0193d2a3a..a854af9def 100644 --- a/config/loader.go +++ b/config/loader.go @@ -60,8 +60,16 @@ func MustLoadProjectConfig(filepath ...string) *ProjectConfig { // 2. filepath. ./optimus -c "path/to/config/optimus.yaml" // 3. current dir. Optimus will look at current directory if there's optimus.yaml there, use it func LoadProjectConfig(filepath ...string) (*ProjectConfig, error) { - cfg := &ProjectConfig{} fs := afero.NewReadOnlyFs(afero.NewOsFs()) + return loadProjectConfigFs(fs, filepath...) +} + +func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ProjectConfig, error) { + cfg := &ProjectConfig{} + + if len(filepath) == 0 { + filepath = append(filepath, "") + } err := loadConfig(cfg, fs, filepath[0], currPath) if err != nil { @@ -87,8 +95,16 @@ func MustLoadServerConfig(filepath ...string) *ServerConfig { // 3. executable binary location // 4. home dir func LoadServerConfig(filepath ...string) (*ServerConfig, error) { - cfg := &ServerConfig{} fs := afero.NewReadOnlyFs(afero.NewOsFs()) + return loadServerConfigFs(fs, filepath...) +} + +func loadServerConfigFs(fs afero.Fs, filepath ...string) (*ServerConfig, error) { + cfg := &ServerConfig{} + + if len(filepath) == 0 { + filepath = append(filepath, "") + } err := loadConfig(cfg, fs, filepath[0], execPath, homePath) if err != nil { @@ -133,10 +149,16 @@ func loadConfig(cfg interface{}, fs afero.Fs, paths ...string) error { } if len(paths) > 0 { - filepath := paths[0] + fpath := paths[0] dirPaths := paths[1:] - opts = append(opts, config.WithFile(filepath)) + if fpath != "" { + if err := validateFilepath(fs, fpath); err != nil { + return err + } + opts = append(opts, config.WithFile(fpath)) + } + for _, path := range dirPaths { opts = append(opts, config.WithPath(path)) } @@ -146,6 +168,17 @@ func loadConfig(cfg interface{}, fs afero.Fs, paths ...string) error { return l.Load(cfg) } +func validateFilepath(fs afero.Fs, fpath string) error { + f, err := fs.Stat(fpath) + if err != nil { + return err + } + if !f.Mode().IsRegular() { + return errors.New("not a file") + } + return nil +} + // type LoadConfigFunc func(interface{}, afero.Fs, ...string) error // // LoadOptimusConfig Load configuration file from following paths diff --git a/config/loader_test.go b/config/loader_test.go index 38185439d4..4ce5ce0e09 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -1,16 +1,18 @@ -package config_test +package config import ( + "io/fs" "os" "path" -) + "strings" + "testing" -const ( - configFileName = ".optimus.yaml" - optimusConfigDirName = "./optimus" + "github.com/spf13/afero" + "github.com/stretchr/testify/suite" ) -const optimusConfigContent = ` +const ( + projectConfig = ` version: 1 log: level: info @@ -21,6 +23,18 @@ project: environment: integration scheduler_host: http://example.io/ storage_path: file://absolute_path_to_a_directory +namespaces: +- name: namespace-a + job: + path: ./jobs-a +- name: namespace-b + job: + path: ./jobs-b +` + serverConfig = ` +version: 1 +log: + level: info serve: port: 9100 host: localhost @@ -39,32 +53,148 @@ scheduler: telemetry: profile_addr: ":9110" jaeger_addr: "http://localhost:14268/api/traces" -namespaces: -- name: namespace-a - job: - path: ./jobs-a -- name: namespace-b - job: - path: ./jobs-b ` -func setup(content string) { - teardown() - if err := os.Mkdir(optimusConfigDirName, 0o750); err != nil { - panic(err) - } - confPath := path.Join(optimusConfigDirName, configFileName) - if err := os.WriteFile(confPath, []byte(content), 0o600); err != nil { - panic(err) - } +type ConfigTestSuite struct { + suite.Suite + a afero.Afero + currPath string + execPath string + homePath string + + expectedProjectConfig *ProjectConfig + expectedServerConfig *ServerConfig } -func teardown() { - if err := os.RemoveAll(optimusConfigDirName); err != nil { - panic(err) +func (s *ConfigTestSuite) SetupTest() { + s.a = afero.Afero{} + s.a.Fs = afero.NewMemMapFs() + + p, err := os.Getwd() + s.Require().NoError(err) + s.currPath = p + s.a.Fs.MkdirAll(s.currPath, fs.ModeTemporary) + + p, err = os.Executable() + s.Require().NoError(err) + s.execPath = p + s.a.Fs.MkdirAll(s.execPath, fs.ModeTemporary) + + p, err = os.UserHomeDir() + s.Require().NoError(err) + s.homePath = p + s.a.Fs.MkdirAll(s.homePath, fs.ModeTemporary) + + s.expectedProjectConfig = &ProjectConfig{} + s.expectedProjectConfig.Version = Version(1) + s.expectedProjectConfig.Log = LogConfig{Level: "info"} + + s.expectedProjectConfig.Host = "localhost:9100" + s.expectedProjectConfig.Project = Project{ + Name: "sample_project", + Config: map[string]string{ + "environment": "integration", + "scheduler_host": "http://example.io/", + "storage_path": "file://absolute_path_to_a_directory", + }, } + namespaces := []*Namespace{} + namespaces = append(namespaces, &Namespace{ + Name: "namespace-a", + Job: Job{ + Path: "./jobs-a", + }, + }) + namespaces = append(namespaces, &Namespace{ + Name: "namespace-b", + Job: Job{ + Path: "./jobs-b", + }, + }) + s.expectedProjectConfig.Namespaces = namespaces +} + +func (s *ConfigTestSuite) TearDownTest() { + s.a.Fs.RemoveAll(s.currPath) + s.a.Fs.RemoveAll(s.execPath) + s.a.Fs.RemoveAll(s.homePath) +} + +func TestConfig(t *testing.T) { + suite.Run(t, new(ConfigTestSuite)) +} + +func (s *ConfigTestSuite) TestInternal_LoadProjectConfigFs() { + s.a.WriteFile(path.Join(s.currPath, filename+"."+fileExtension), []byte(projectConfig), fs.ModeTemporary) + + s.Run("WhenFilepathIsEmpty", func() { + p, err := loadProjectConfigFs(s.a.Fs) + s.Assert().NoError(err) + s.Assert().NotNil(p) + s.Assert().Equal(s.expectedProjectConfig, p) + }) + + s.Run("WhenFilepathIsExist", func() { + samplePath := "./sample/path/config.yaml" + b := strings.Builder{} + b.WriteString(projectConfig) + b.WriteString(`- name: namespace-c + job: + path: ./jobs-c + `) + s.a.WriteFile(samplePath, []byte(b.String()), fs.ModeTemporary) + defer s.a.Fs.RemoveAll(samplePath) + + p, err := loadProjectConfigFs(s.a.Fs, samplePath) + s.Assert().NoError(err) + s.Assert().NotNil(p) + s.Assert().Len(p.Namespaces, 3) + }) + + s.Run("WhenLoadConfigIsFailed", func() { + p, err := loadProjectConfigFs(s.a.Fs, "/path/not/exist") + s.Assert().Error(err) + s.Assert().Nil(p) + }) } +func (s *ConfigTestSuite) TestInternal_LoadServerConfigFs() { + // TODO: implement this +} + +func (s *ConfigTestSuite) TestLoadProjectConfig() { + // TODO: implement this +} + +func (s *ConfigTestSuite) TestMustLoadProjectConfig() { + // TODO: implement this +} + +func (s *ConfigTestSuite) TestLoadServerConfig() { + // TODO: implement this +} + +func (s *ConfigTestSuite) TestMustLoadServerConfig() { + // TODO: implement this +} + +// func setup(content string) { +// teardown() +// if err := os.Mkdir(optimusConfigDirName, os.ModePerm); err != nil { +// panic(err) +// } +// confPath := path.Join(optimusConfigDirName, configFileName) +// if err := os.WriteFile(confPath, []byte(content), os.ModePerm); err != nil { +// panic(err) +// } +// } + +// func teardown() { +// if err := os.RemoveAll(optimusConfigDirName); err != nil { +// panic(err) +// } +// } + // func TestLoadOptimusConfig(t *testing.T) { // t.Run("should return config and nil if no error is found", func(t *testing.T) { // setup(optimusConfigContent + ` From 8ed82789753c156484dd12da8f256a1c167bc2b5 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 22 Mar 2022 19:25:11 +0700 Subject: [PATCH 05/54] test: add test for load server config fs --- config/loader_test.go | 223 ++++++++++++++++++------------------------ 1 file changed, 96 insertions(+), 127 deletions(-) diff --git a/config/loader_test.go b/config/loader_test.go index 4ce5ce0e09..9b288e6361 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -6,13 +6,13 @@ import ( "path" "strings" "testing" + "time" "github.com/spf13/afero" "github.com/stretchr/testify/suite" ) -const ( - projectConfig = ` +const projectConfig = ` version: 1 log: level: info @@ -31,7 +31,7 @@ namespaces: job: path: ./jobs-b ` - serverConfig = ` +const serverConfig = ` version: 1 log: level: info @@ -85,33 +85,8 @@ func (s *ConfigTestSuite) SetupTest() { s.homePath = p s.a.Fs.MkdirAll(s.homePath, fs.ModeTemporary) - s.expectedProjectConfig = &ProjectConfig{} - s.expectedProjectConfig.Version = Version(1) - s.expectedProjectConfig.Log = LogConfig{Level: "info"} - - s.expectedProjectConfig.Host = "localhost:9100" - s.expectedProjectConfig.Project = Project{ - Name: "sample_project", - Config: map[string]string{ - "environment": "integration", - "scheduler_host": "http://example.io/", - "storage_path": "file://absolute_path_to_a_directory", - }, - } - namespaces := []*Namespace{} - namespaces = append(namespaces, &Namespace{ - Name: "namespace-a", - Job: Job{ - Path: "./jobs-a", - }, - }) - namespaces = append(namespaces, &Namespace{ - Name: "namespace-b", - Job: Job{ - Path: "./jobs-b", - }, - }) - s.expectedProjectConfig.Namespaces = namespaces + s.initExpectedProjectConfig() + s.initExpectedServerConfig() } func (s *ConfigTestSuite) TearDownTest() { @@ -159,7 +134,41 @@ func (s *ConfigTestSuite) TestInternal_LoadProjectConfigFs() { } func (s *ConfigTestSuite) TestInternal_LoadServerConfigFs() { - // TODO: implement this + s.a.WriteFile(path.Join(s.execPath, filename+"."+fileExtension), []byte(serverConfig), fs.ModeTemporary) + s.a.WriteFile(path.Join(s.homePath, filename+"."+fileExtension), []byte(`version: 3`), fs.ModeTemporary) + + s.Run("WhenFilepathIsEmpty", func() { + conf, err := loadServerConfigFs(s.a.Fs) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal(s.expectedServerConfig, conf) // should load from exec path + + s.a.Remove(path.Join(s.execPath, filename+"."+fileExtension)) + defer s.a.WriteFile(path.Join(s.execPath, filename+"."+fileExtension), []byte(serverConfig), fs.ModeTemporary) + conf, err = loadServerConfigFs(s.a.Fs) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal("3", conf.Version.String()) // should load from home dir + }) + + s.Run("WhenFilepathIsExist", func() { + samplePath := "./sample/path/config.yaml" + s.a.WriteFile(samplePath, []byte("version: 2"), os.ModeTemporary) + + conf, err := loadServerConfigFs(s.a.Fs, samplePath) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal("2", conf.Version.String()) + }) + + s.Run("WhenLoadConfigIsFailed", func() { + dirPath := "./sample/path" + s.a.Fs.MkdirAll(dirPath, os.ModeTemporary) + + conf, err := loadServerConfigFs(s.a.Fs, dirPath) + s.Assert().Error(err) + s.Assert().Nil(conf) + }) } func (s *ConfigTestSuite) TestLoadProjectConfig() { @@ -178,99 +187,59 @@ func (s *ConfigTestSuite) TestMustLoadServerConfig() { // TODO: implement this } -// func setup(content string) { -// teardown() -// if err := os.Mkdir(optimusConfigDirName, os.ModePerm); err != nil { -// panic(err) -// } -// confPath := path.Join(optimusConfigDirName, configFileName) -// if err := os.WriteFile(confPath, []byte(content), os.ModePerm); err != nil { -// panic(err) -// } -// } - -// func teardown() { -// if err := os.RemoveAll(optimusConfigDirName); err != nil { -// panic(err) -// } -// } - -// func TestLoadOptimusConfig(t *testing.T) { -// t.Run("should return config and nil if no error is found", func(t *testing.T) { -// setup(optimusConfigContent + ` -// - name: namespace-b -// job: -// path: ./jobs-b -// `) -// defer teardown() - -// expectedErrMsg := "namespaces [namespace-b] are duplicate" - -// actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) - -// assert.Nil(t, actualConf) -// assert.EqualError(t, actualErr, expectedErrMsg) -// }) - -// t.Run("should return config and nil if no error is found", func(t *testing.T) { -// setup(optimusConfigContent) -// defer teardown() - -// expectedConf := &config.Optimus{ -// Version: 1, -// Log: config.LogConfig{ -// Level: "info", -// }, -// Host: "localhost:9100", -// Project: config.Project{ -// Name: "sample_project", -// Config: map[string]string{ -// "environment": "integration", -// "scheduler_host": "http://example.io/", -// "storage_path": "file://absolute_path_to_a_directory", -// }, -// }, -// Server: config.ServerConfig{ -// Port: 9100, -// Host: "localhost", -// IngressHost: "optimus.example.io:80", -// AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", -// ReplayNumWorkers: 1, -// ReplayWorkerTimeout: 100 * time.Second, -// ReplayRunTimeout: 10 * time.Second, -// DB: config.DBConfig{ -// DSN: "postgres://user:password@localhost:5432/database?sslmode=disable", -// MaxIdleConnection: 5, -// MaxOpenConnection: 10, -// }, -// }, -// Scheduler: config.SchedulerConfig{ -// Name: "airflow2", -// SkipInit: true, -// }, -// Telemetry: config.TelemetryConfig{ -// ProfileAddr: ":9110", -// JaegerAddr: "http://localhost:14268/api/traces", -// }, -// Namespaces: []*config.Namespace{ -// { -// Name: "namespace-a", -// Job: config.Job{ -// Path: "./jobs-a", -// }, -// }, -// { -// Name: "namespace-b", -// Job: config.Job{ -// Path: "./jobs-b", -// }, -// }, -// }, -// } - -// actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) - -// assert.EqualValues(t, expectedConf, actualConf) -// assert.NoError(t, actualErr) -// }) -// } +func (s *ConfigTestSuite) initExpectedProjectConfig() { + s.expectedProjectConfig = &ProjectConfig{} + s.expectedProjectConfig.Version = Version(1) + s.expectedProjectConfig.Log = LogConfig{Level: "info"} + + s.expectedProjectConfig.Host = "localhost:9100" + s.expectedProjectConfig.Project = Project{ + Name: "sample_project", + Config: map[string]string{ + "environment": "integration", + "scheduler_host": "http://example.io/", + "storage_path": "file://absolute_path_to_a_directory", + }, + } + namespaces := []*Namespace{} + namespaces = append(namespaces, &Namespace{ + Name: "namespace-a", + Job: Job{ + Path: "./jobs-a", + }, + }) + namespaces = append(namespaces, &Namespace{ + Name: "namespace-b", + Job: Job{ + Path: "./jobs-b", + }, + }) + s.expectedProjectConfig.Namespaces = namespaces +} + +func (s *ConfigTestSuite) initExpectedServerConfig() { + s.expectedServerConfig = &ServerConfig{} + s.expectedServerConfig.Version = Version(1) + s.expectedServerConfig.Log = LogConfig{Level: "info"} + + s.expectedServerConfig.Serve = Serve{} + s.expectedServerConfig.Serve.Port = 9100 + s.expectedServerConfig.Serve.Host = "localhost" + s.expectedServerConfig.Serve.IngressHost = "optimus.example.io:80" + s.expectedServerConfig.Serve.AppKey = "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc" + s.expectedServerConfig.Serve.ReplayNumWorkers = 1 + s.expectedServerConfig.Serve.ReplayWorkerTimeout = 100 * time.Second + s.expectedServerConfig.Serve.ReplayRunTimeout = 10 * time.Second + s.expectedServerConfig.Serve.DB = DBConfig{} + s.expectedServerConfig.Serve.DB.DSN = "postgres://user:password@localhost:5432/database?sslmode=disable" + s.expectedServerConfig.Serve.DB.MaxIdleConnection = 5 + s.expectedServerConfig.Serve.DB.MaxOpenConnection = 10 + + s.expectedServerConfig.Scheduler = SchedulerConfig{} + s.expectedServerConfig.Scheduler.Name = "airflow2" + s.expectedServerConfig.Scheduler.SkipInit = true + + s.expectedServerConfig.Telemetry = TelemetryConfig{} + s.expectedServerConfig.Telemetry.ProfileAddr = ":9110" + s.expectedServerConfig.Telemetry.JaegerAddr = "http://localhost:14268/api/traces" +} From b7e94403b5a9b062369b2c8d4c5cf588c28fcead Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 22 Mar 2022 19:30:04 +0700 Subject: [PATCH 06/54] test: add test for load server and project config --- config/loader_test.go | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/config/loader_test.go b/config/loader_test.go index 9b288e6361..fdcf90f74d 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path" + "path/filepath" "strings" "testing" "time" @@ -172,7 +173,15 @@ func (s *ConfigTestSuite) TestInternal_LoadServerConfigFs() { } func (s *ConfigTestSuite) TestLoadProjectConfig() { - // TODO: implement this + path := os.TempDir() + fpath := filepath.Join(path, filename+"."+fileExtension) + defer os.Remove(fpath) + os.WriteFile(fpath, []byte(projectConfig), 0400) + + conf, err := LoadProjectConfig(fpath) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal(s.expectedProjectConfig, conf) } func (s *ConfigTestSuite) TestMustLoadProjectConfig() { @@ -180,7 +189,15 @@ func (s *ConfigTestSuite) TestMustLoadProjectConfig() { } func (s *ConfigTestSuite) TestLoadServerConfig() { - // TODO: implement this + path := os.TempDir() + fpath := filepath.Join(path, filename+"."+fileExtension) + defer os.Remove(fpath) + os.WriteFile(fpath, []byte(serverConfig), 0400) + + conf, err := LoadServerConfig(fpath) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal(s.expectedServerConfig, conf) } func (s *ConfigTestSuite) TestMustLoadServerConfig() { From 7cc07b2b96401d1a2bb51225f88cdb4e2afbec82 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 22 Mar 2022 19:36:13 +0700 Subject: [PATCH 07/54] refactor: remove unecessary code --- config/loader.go | 68 ------------------------------------------- config/loader_test.go | 8 ----- 2 files changed, 76 deletions(-) diff --git a/config/loader.go b/config/loader.go index a854af9def..f94d3360b1 100644 --- a/config/loader.go +++ b/config/loader.go @@ -45,16 +45,6 @@ func init() { homePath = p } -// MustLoadProjectConfig load the project specific config (see LoadProjectConfig) with panic -func MustLoadProjectConfig(filepath ...string) *ProjectConfig { - cfg, err := LoadProjectConfig(filepath...) - if err != nil { - panic(err) - } - - return cfg -} - // LoadProjectConfig load the project specific config from these locations: // 1. env var. eg. OPTIMUS_PROJECT, OPTIMUS_NAMESPACES, etc // 2. filepath. ./optimus -c "path/to/config/optimus.yaml" @@ -79,16 +69,6 @@ func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ProjectConfig, error return cfg, nil } -// MustLoadServerConfig load the server specific config (see LoadServerConfig) with panic -func MustLoadServerConfig(filepath ...string) *ServerConfig { - cfg, err := LoadServerConfig(filepath...) - if err != nil { - panic(err) - } - - return cfg -} - // LoadServerConfig load the server specific config from these locations: // 1. env var. eg. OPTIMUS_SERVE_PORT, etc // 2. filepath. ./optimus -c "path/to/config.yaml" @@ -178,51 +158,3 @@ func validateFilepath(fs afero.Fs, fpath string) error { } return nil } - -// type LoadConfigFunc func(interface{}, afero.Fs, ...string) error - -// // LoadOptimusConfig Load configuration file from following paths -// // ./ -// // / -// // ~/.optimus/ -// // Namespaces will be loaded only from current project ./ -// func LoadOptimusConfig(dirPaths ...string) (*Optimus, error) { -// fs := afero.NewReadOnlyFs(afero.NewOsFs()) - -// var targetPath string -// if len(dirPaths) > 0 { -// targetPath = dirPaths[0] -// } else { -// currPath, err := os.Getwd() -// if err != nil { -// return nil, fmt.Errorf("error getting current work directory path: %w", err) -// } -// targetPath = currPath -// } - -// optimus := Optimus{} -// if err := loadConfig(&optimus, fs, targetPath); err != nil { -// return nil, errors.New("error loading config") -// } -// if err := validateNamespaceDuplication(&optimus); err != nil { -// return nil, err -// } -// return &optimus, nil -// } - -// func validateNamespaceDuplication(optimus *Optimus) error { -// nameToAppearance := make(map[string]int) -// for _, namespace := range optimus.Namespaces { -// nameToAppearance[namespace.Name]++ -// } -// var duplicateNames []string -// for name, appearance := range nameToAppearance { -// if appearance > 1 { -// duplicateNames = append(duplicateNames, name) -// } -// } -// if len(duplicateNames) > 0 { -// return fmt.Errorf("namespaces [%s] are duplicate", strings.Join(duplicateNames, ", ")) -// } -// return nil -// } diff --git a/config/loader_test.go b/config/loader_test.go index fdcf90f74d..2d21da8139 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -184,10 +184,6 @@ func (s *ConfigTestSuite) TestLoadProjectConfig() { s.Assert().Equal(s.expectedProjectConfig, conf) } -func (s *ConfigTestSuite) TestMustLoadProjectConfig() { - // TODO: implement this -} - func (s *ConfigTestSuite) TestLoadServerConfig() { path := os.TempDir() fpath := filepath.Join(path, filename+"."+fileExtension) @@ -200,10 +196,6 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { s.Assert().Equal(s.expectedServerConfig, conf) } -func (s *ConfigTestSuite) TestMustLoadServerConfig() { - // TODO: implement this -} - func (s *ConfigTestSuite) initExpectedProjectConfig() { s.expectedProjectConfig = &ProjectConfig{} s.expectedProjectConfig.Version = Version(1) From d081d55af985aa555d395975ce1f304609c35e4f Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 09:54:00 +0700 Subject: [PATCH 08/54] feat: add config validator --- config/loader.go | 22 +------- config/validation.go | 63 +++++++++++++++++++++ config/validation_test.go | 116 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 180 insertions(+), 21 deletions(-) create mode 100644 config/validation.go create mode 100644 config/validation_test.go diff --git a/config/loader.go b/config/loader.go index f94d3360b1..722bc3e80e 100644 --- a/config/loader.go +++ b/config/loader.go @@ -6,6 +6,7 @@ import ( "github.com/odpf/salt/config" "github.com/spf13/afero" + "github.com/spf13/viper" ) @@ -94,27 +95,6 @@ func loadServerConfigFs(fs afero.Fs, filepath ...string) (*ServerConfig, error) return cfg, nil } -// Validate validate the config as an input. If not valid, it returns error -func Validate(conf interface{}) error { - switch c := conf.(type) { - case ProjectConfig: - return validateProjectConfig(c) - case ServerConfig: - return validateServerConfig(c) - } - return errors.New("error") -} - -func validateProjectConfig(conf ProjectConfig) error { - // implement this - return nil -} - -func validateServerConfig(conf ServerConfig) error { - // implement this - return nil -} - func loadConfig(cfg interface{}, fs afero.Fs, paths ...string) error { // getViperWithDefault + SetFs v := viper.New() diff --git a/config/validation.go b/config/validation.go new file mode 100644 index 0000000000..80362a7248 --- /dev/null +++ b/config/validation.go @@ -0,0 +1,63 @@ +package config + +import ( + "errors" + "fmt" + "strings" + + validation "github.com/go-ozzo/ozzo-validation/v4" +) + +// Validate validate the config as an input. If not valid, it returns error +func Validate(conf interface{}) error { + switch c := conf.(type) { + case ProjectConfig: + return validateProjectConfig(c) + case ServerConfig: + return validateServerConfig(c) + } + return errors.New("error") +} + +func validateProjectConfig(conf ProjectConfig) error { + // implement this + return validation.ValidateStruct(&conf, + validation.Field(&conf.Version, validation.Required), + validation.Field(&conf.Host, validation.Required), + validation.Field(&conf.Namespaces, validation.By(validateNamespaces)), + // ... etc + ) +} + +func validateServerConfig(conf ServerConfig) error { + // implement this + return nil +} + +func validateNamespaces(value interface{}) error { + namespaces, ok := value.([]*Namespace) + if !ok { + return errors.New("error") + } + + m := map[string]int{} + for _, n := range namespaces { + if n == nil { + continue + } + m[n.Name]++ + } + + dup := []string{} + for k, v := range m { + if v > 1 { + dup = append(dup, k) + } + } + + if len(dup) > 0 { + return fmt.Errorf("duplicate namespaces are not allowed [%s]", strings.Join(dup, ",")) + } + + return nil +} diff --git a/config/validation_test.go b/config/validation_test.go new file mode 100644 index 0000000000..13e3c569ff --- /dev/null +++ b/config/validation_test.go @@ -0,0 +1,116 @@ +package config + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type ValidationTestSuite struct { + suite.Suite + defaultProjectConfig ProjectConfig +} + +func (s *ValidationTestSuite) SetupTest() { + s.initDefaultProjectConfig() +} + +func TestValidation(t *testing.T) { + suite.Run(t, new(ValidationTestSuite)) +} + +func (s *ValidationTestSuite) TestInternal_ValidateNamespaces_Success() { + namespaces := []*Namespace{ + { + Name: "namespace-1", + Job: Job{Path: "path-1"}, + }, { + Name: "namespace-2", + Job: Job{Path: "path-2"}, + }, + nil, + } + + err := validateNamespaces(namespaces) + s.Assert().NoError(err) +} + +func (s *ValidationTestSuite) TestInternal_ValidateNamespaces_Fail() { + s.Run("WhenTypeAssertionIsFailed", func() { + invalidStruct := "this-is-string" + err := validateNamespaces(invalidStruct) + s.Assert().Error(err) + }) + + s.Run("WhenDuplicationIsDetected", func() { + namespaces := []*Namespace{ + { + Name: "other-ns", + Job: Job{Path: "path-other"}, + }, { + Name: "dup-ns", + Job: Job{Path: "path-1"}, + }, { + Name: "dup-ns", + Job: Job{Path: "path-2"}, + }, + } + + err := validateNamespaces(namespaces) + s.Assert().Error(err) + }) +} + +func (s *ValidationTestSuite) TestValidate_ProjectConfig() { + s.Run("WhenStructIsValid", func() { + err := Validate(s.defaultProjectConfig) + s.Assert().NoError(err) + }) + + s.Run("WhenStructIsInvalid", func() { + conf := s.defaultProjectConfig + conf.Host = "" + + err := Validate(conf) + s.Assert().Error(err) + }) +} + +func (s *ValidationTestSuite) TestValidate_ServerConfig() { + // TODO: implement this +} + +func (s *ValidationTestSuite) TestValidate_Fail() { + err := Validate("invalid-type") + s.Assert().Error(err) +} + +func (s *ValidationTestSuite) initDefaultProjectConfig() { + s.defaultProjectConfig = ProjectConfig{} + s.defaultProjectConfig.Version = Version(1) + s.defaultProjectConfig.Log = LogConfig{Level: "info"} + + s.defaultProjectConfig.Host = "localhost:9100" + s.defaultProjectConfig.Project = Project{ + Name: "sample_project", + Config: map[string]string{ + "environment": "integration", + "scheduler_host": "http://example.io/", + "storage_path": "file://absolute_path_to_a_directory", + }, + } + namespaces := []*Namespace{} + namespaces = append(namespaces, &Namespace{ + Name: "namespace-a", + Job: Job{ + Path: "./jobs-a", + }, + }) + namespaces = append(namespaces, &Namespace{ + Name: "namespace-b", + Job: Job{ + Path: "./jobs-b", + }, + }) + s.defaultProjectConfig.Namespaces = namespaces +} From 8a133add44a0bdf24b5cf42f5186309588f86995 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 10:06:06 +0700 Subject: [PATCH 09/54] refactor: loader --- config/loader.go | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/config/loader.go b/config/loader.go index 722bc3e80e..4a83c29d08 100644 --- a/config/loader.go +++ b/config/loader.go @@ -55,6 +55,16 @@ func LoadProjectConfig(filepath ...string) (*ProjectConfig, error) { return loadProjectConfigFs(fs, filepath...) } +// LoadServerConfig load the server specific config from these locations: +// 1. env var. eg. OPTIMUS_SERVE_PORT, etc +// 2. filepath. ./optimus -c "path/to/config.yaml" +// 3. executable binary location +// 4. home dir +func LoadServerConfig(filepath ...string) (*ServerConfig, error) { + fs := afero.NewReadOnlyFs(afero.NewOsFs()) + return loadServerConfigFs(fs, filepath...) +} + func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ProjectConfig, error) { cfg := &ProjectConfig{} @@ -70,16 +80,6 @@ func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ProjectConfig, error return cfg, nil } -// LoadServerConfig load the server specific config from these locations: -// 1. env var. eg. OPTIMUS_SERVE_PORT, etc -// 2. filepath. ./optimus -c "path/to/config.yaml" -// 3. executable binary location -// 4. home dir -func LoadServerConfig(filepath ...string) (*ServerConfig, error) { - fs := afero.NewReadOnlyFs(afero.NewOsFs()) - return loadServerConfigFs(fs, filepath...) -} - func loadServerConfigFs(fs afero.Fs, filepath ...string) (*ServerConfig, error) { cfg := &ServerConfig{} From f00cc3b4c84735a5102b931578b8e58fbd3501bd Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 10:43:41 +0700 Subject: [PATCH 10/54] refactor: load config inside specific command --- cmd/admin.go | 14 ++++++++--- cmd/admin_build_instance.go | 2 +- cmd/backup.go | 16 +++++++++--- cmd/backup_create.go | 2 +- cmd/backup_list.go | 2 +- cmd/backup_status.go | 2 +- cmd/commands.go | 27 ++++++--------------- cmd/config.go | 8 +++--- cmd/deploy.go | 25 ++++++++++++++++--- cmd/job.go | 24 +++++++++++------- cmd/job_create.go | 2 +- cmd/job_hook.go | 2 +- cmd/job_render.go | 2 +- cmd/job_run.go | 9 +++++-- cmd/job_validate.go | 4 ++- cmd/namespace.go | 2 +- cmd/replay.go | 16 +++++++++--- cmd/replay_create.go | 2 +- cmd/replay_list.go | 2 +- cmd/replay_status.go | 6 +++-- cmd/resource.go | 24 +++++++++++++++--- cmd/secret.go | 22 +++++++++++------ cmd/serve.go | 12 +++++++-- cmd/server/server.go | 2 +- cmd/version.go | 1 + config/config_project.go | 20 +++++++++++++++ ext/scheduler/airflow2/compiler/compiler.go | 2 +- go.mod | 5 +++- go.sum | 4 +++ 29 files changed, 184 insertions(+), 77 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index a07bc2bb7f..8582184ee9 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -8,18 +8,26 @@ import ( ) // adminCommand registers internal administration commands -func adminCommand(l log.Logger, conf config.Optimus) *cli.Command { +func adminCommand(l log.Logger) *cli.Command { cmd := &cli.Command{ Use: "admin", Short: "Internal administration commands", Hidden: true, } - cmd.AddCommand(adminBuildCommand(l, conf)) + + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + cmd.AddCommand(adminBuildCommand(l, *conf)) return cmd } // adminBuildCommand builds a run instance -func adminBuildCommand(l log.Logger, conf config.Optimus) *cli.Command { +func adminBuildCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { cmd := &cli.Command{ Use: "build", Short: "Register a job run and get required assets", diff --git a/cmd/admin_build_instance.go b/cmd/admin_build_instance.go index 6a1b329784..8b01712c3b 100644 --- a/cmd/admin_build_instance.go +++ b/cmd/admin_build_instance.go @@ -27,7 +27,7 @@ const ( const unsubstitutedValue = "" -func adminBuildInstanceCommand(l log.Logger, conf config.Optimus) *cli.Command { +func adminBuildInstanceCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { var ( optimusHost = conf.Host projectName = conf.Project.Name diff --git a/cmd/backup.go b/cmd/backup.go index 51fb26b9ce..e1a4aafb9e 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -15,7 +15,7 @@ const ( backupTimeout = time.Minute * 15 ) -func backupCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Command { cmd := &cli.Command{ Use: "backup", Short: "Backup a resource and its downstream", @@ -27,8 +27,16 @@ func backupCommand(l log.Logger, conf config.Optimus, datastoreRepo models.Datas "group:core": "true", }, } - cmd.AddCommand(backupCreateCommand(l, conf, datastoreRepo)) - cmd.AddCommand(backupListCommand(l, conf, datastoreRepo)) - cmd.AddCommand(backupStatusCommand(l, conf, datastoreRepo)) + + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + cmd.AddCommand(backupCreateCommand(l, *conf, datastoreRepo)) + cmd.AddCommand(backupListCommand(l, *conf, datastoreRepo)) + cmd.AddCommand(backupStatusCommand(l, *conf, datastoreRepo)) return cmd } diff --git a/cmd/backup_create.go b/cmd/backup_create.go index 23793da4f0..a738093f2d 100644 --- a/cmd/backup_create.go +++ b/cmd/backup_create.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupCreateCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCreateCommand(l log.Logger, conf config.ProjectConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( backupCmd = &cli.Command{ Use: "create", diff --git a/cmd/backup_list.go b/cmd/backup_list.go index fccf8bd6ee..d851b4c43e 100644 --- a/cmd/backup_list.go +++ b/cmd/backup_list.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupListCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupListCommand(l log.Logger, conf config.ProjectConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( backupCmd = &cli.Command{ Use: "list", diff --git a/cmd/backup_status.go b/cmd/backup_status.go index e749526cc8..9671c7dbd4 100644 --- a/cmd/backup_status.go +++ b/cmd/backup_status.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupStatusCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupStatusCommand(l log.Logger, conf config.ProjectConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( project string backupCmd = &cli.Command{ diff --git a/cmd/commands.go b/cmd/commands.go index 21761e2944..51ed3a1131 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -17,7 +17,6 @@ import ( "github.com/odpf/salt/cmdx" "github.com/odpf/salt/log" "github.com/odpf/salt/term" - "github.com/spf13/afero" cli "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" @@ -110,26 +109,16 @@ func New(plainLog log.Logger, jsonLog log.Logger, pluginRepo models.PluginReposi cmdx.SetHelp(cmd) cmd.PersistentFlags().BoolVar(&disableColoredOut, "no-color", disableColoredOut, "Disable colored output") - // init local specs - datastoreSpecFs := make(map[string]map[string]afero.Fs) - for _, namespace := range conf.Namespaces { - dtSpec := make(map[string]afero.Fs) - for _, dsConfig := range namespace.Datastore { - dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) - } - datastoreSpecFs[namespace.Name] = dtSpec - } - cmd.AddCommand(versionCommand(plainLog, pluginRepo)) cmd.AddCommand(configCommand(plainLog)) - cmd.AddCommand(jobCommand(plainLog, conf, pluginRepo)) - cmd.AddCommand(deployCommand(plainLog, conf, pluginRepo, dsRepo, datastoreSpecFs)) - cmd.AddCommand(resourceCommand(plainLog, conf, dsRepo, datastoreSpecFs)) - cmd.AddCommand(serveCommand(jsonLog, conf)) - cmd.AddCommand(replayCommand(plainLog, conf)) - cmd.AddCommand(backupCommand(plainLog, conf, dsRepo)) - cmd.AddCommand(adminCommand(plainLog, conf)) - cmd.AddCommand(secretCommand(plainLog, conf)) + cmd.AddCommand(jobCommand(plainLog, pluginRepo)) + cmd.AddCommand(deployCommand(plainLog, pluginRepo, dsRepo)) + cmd.AddCommand(resourceCommand(plainLog, dsRepo)) + cmd.AddCommand(serveCommand(jsonLog)) + cmd.AddCommand(replayCommand(plainLog)) + cmd.AddCommand(backupCommand(plainLog, dsRepo)) + cmd.AddCommand(adminCommand(plainLog)) + cmd.AddCommand(secretCommand(plainLog)) addExtensionCommand(cmd, plainLog) return cmd diff --git a/cmd/config.go b/cmd/config.go index 8f1f84b0e0..caad5c7187 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -30,8 +30,8 @@ func configInitCommand(l log.Logger) *cli.Command { Use: "init", Short: "Initialize optimus configuration file", RunE: func(c *cli.Command, args []string) (err error) { - conf := config.Optimus{ - Version: 1, + conf := config.ProjectConfig{ + Version: config.Version(1), Host: defaultHost, } questions := []*survey.Question{ @@ -93,7 +93,7 @@ func configInitCommand(l log.Logger) *cli.Command { if err != nil { return err } - if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.FileName, config.FileExtension), confMarshaled, 0o600); err != nil { + if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension), confMarshaled, 0655); err != nil { return err } l.Info(coloredSuccess("Configuration initialised successfully")) @@ -103,7 +103,7 @@ func configInitCommand(l log.Logger) *cli.Command { return c } -func projectConfigQuestions(conf config.Optimus) (config.Optimus, error) { +func projectConfigQuestions(conf config.ProjectConfig) (config.ProjectConfig, error) { conf.Project.Config = map[string]string{} registerMore := AnswerYes for registerMore == AnswerNo { diff --git a/cmd/deploy.go b/cmd/deploy.go index 154da5e5fc..d8d24b964f 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -27,8 +27,7 @@ const ( ) // deployCommand pushes current repo to optimus service -func deployCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo, - datastoreSpecFs map[string]map[string]afero.Fs) *cli.Command { +func deployCommand(l log.Logger, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { var ( namespaces []string ignoreJobs bool @@ -45,6 +44,24 @@ func deployCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRe }, } ) + + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + //init local specs + datastoreSpecFs := make(map[string]map[string]afero.Fs) + for _, namespace := range conf.Namespaces { + dtSpec := make(map[string]afero.Fs) + for _, dsConfig := range namespace.Datastore { + dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) + } + datastoreSpecFs[namespace.Name] = dtSpec + } + cmd.Flags().StringSliceVarP(&namespaces, "namespaces", "N", nil, "Selected namespaces of optimus project") cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Print details related to deployment stages") cmd.Flags().BoolVar(&ignoreJobs, "ignore-jobs", false, "Ignore deployment of jobs") @@ -57,7 +74,7 @@ func deployCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRe if err := validateNamespaces(datastoreSpecFs, namespaces); err != nil { return err } - err := postDeploymentRequest(l, conf, pluginRepo, dsRepo, datastoreSpecFs, namespaces, ignoreJobs, ignoreResources, verbose) + err := postDeploymentRequest(l, *conf, pluginRepo, dsRepo, datastoreSpecFs, namespaces, ignoreJobs, ignoreResources, verbose) if err != nil { return err } @@ -69,7 +86,7 @@ func deployCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRe } // postDeploymentRequest send a deployment request to service -func postDeploymentRequest(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, +func postDeploymentRequest(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository, datastoreRepo models.DatastoreRepo, datastoreSpecFs map[string]map[string]afero.Fs, namespaceNames []string, ignoreJobDeployment, ignoreResources, verbose bool) error { dialTimeoutCtx, dialCancel := context.WithTimeout(context.Background(), OptimusDialTimeout) diff --git a/cmd/job.go b/cmd/job.go index b7306bfae5..dd77ac7863 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -1,14 +1,13 @@ package cmd import ( - "github.com/odpf/salt/log" - cli "github.com/spf13/cobra" - "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" + "github.com/odpf/salt/log" + cli "github.com/spf13/cobra" ) -func jobCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository) *cli.Command { +func jobCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "job", Short: "Interact with schedulable Job", @@ -17,11 +16,18 @@ func jobCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepos }, } - cmd.AddCommand(jobCreateCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobAddHookCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobRenderTemplateCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobValidateCommand(l, conf, pluginRepo, conf.Project.Name, conf.Host)) - cmd.AddCommand(jobRunCommand(l, conf, pluginRepo, conf.Project.Name, conf.Host)) + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + cmd.AddCommand(jobCreateCommand(l, *conf, pluginRepo)) + cmd.AddCommand(jobAddHookCommand(l, *conf, pluginRepo)) + cmd.AddCommand(jobRenderTemplateCommand(l, *conf, pluginRepo)) + cmd.AddCommand(jobValidateCommand(l, *conf, pluginRepo)) + cmd.AddCommand(jobRunCommand(l, *conf, pluginRepo)) cmd.AddCommand(jobRunListCommand(l, conf.Project.Name, conf.Host)) return cmd } diff --git a/cmd/job_create.go b/cmd/job_create.go index a7527bb30f..40852c8650 100644 --- a/cmd/job_create.go +++ b/cmd/job_create.go @@ -33,7 +33,7 @@ var ( specFileNames = []string{local.ResourceSpecFileName, local.JobSpecFileName} ) -func jobCreateCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository) *cli.Command { +func jobCreateCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new Job", diff --git a/cmd/job_hook.go b/cmd/job_hook.go index c4f8852f4f..074edb731e 100644 --- a/cmd/job_hook.go +++ b/cmd/job_hook.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobAddHookCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository) *cli.Command { +func jobAddHookCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "addhook", Aliases: []string{"add_hook", "add-hook", "addHook", "attach_hook", "attach-hook", "attachHook"}, diff --git a/cmd/job_render.go b/cmd/job_render.go index 449734e2d2..ad45b275a4 100644 --- a/cmd/job_render.go +++ b/cmd/job_render.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobRenderTemplateCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository) *cli.Command { +func jobRenderTemplateCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "render", Short: "Apply template values in job specification to current 'render' directory", diff --git a/cmd/job_run.go b/cmd/job_run.go index 606249945b..b38e931e3f 100644 --- a/cmd/job_run.go +++ b/cmd/job_run.go @@ -22,8 +22,13 @@ const ( runJobTimeout = time.Minute * 1 ) -func jobRunCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, projectName, host string) *cli.Command { - var namespaceName string +func jobRunCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { + var ( + namespaceName string + projectName = conf.Project.Name + host = conf.Host + ) + cmd := &cli.Command{ Use: "run", Short: "[EXPERIMENTAL] run the provided job on optimus cluster", diff --git a/cmd/job_validate.go b/cmd/job_validate.go index e2f10f6fc6..06d1be1fbe 100644 --- a/cmd/job_validate.go +++ b/cmd/job_validate.go @@ -23,10 +23,12 @@ const ( validateTimeout = time.Minute * 5 ) -func jobValidateCommand(l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, projectName, host string) *cli.Command { +func jobValidateCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { var ( verbose bool namespaceName string + projectName = conf.Project.Name + host = conf.Host ) cmd := &cli.Command{ Use: "validate", diff --git a/cmd/namespace.go b/cmd/namespace.go index 48cc112447..592fb7720c 100644 --- a/cmd/namespace.go +++ b/cmd/namespace.go @@ -9,7 +9,7 @@ import ( "github.com/odpf/optimus/config" ) -func askToSelectNamespace(l log.Logger, conf config.Optimus) (*config.Namespace, error) { +func askToSelectNamespace(l log.Logger, conf config.ProjectConfig) (*config.Namespace, error) { options := make([]string, len(conf.Namespaces)) if len(conf.Namespaces) == 0 { return nil, errors.New("no namespace found in config file") diff --git a/cmd/replay.go b/cmd/replay.go index ff439ddcd1..02cb7e7497 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -52,7 +52,7 @@ func formatRunsPerJobInstance(instance *pb.ReplayExecutionTreeNode, taskReruns m } } -func replayCommand(l log.Logger, conf config.Optimus) *cli.Command { +func replayCommand(l log.Logger) *cli.Command { cmd := &cli.Command{ Use: "replay", Short: "Re-running jobs in order to update data for older dates/partitions", @@ -61,8 +61,16 @@ func replayCommand(l log.Logger, conf config.Optimus) *cli.Command { "group:core": "true", }, } - cmd.AddCommand(replayCreateCommand(l, conf)) - cmd.AddCommand(replayStatusCommand(l, conf)) - cmd.AddCommand(replayListCommand(l, conf)) + + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + cmd.AddCommand(replayCreateCommand(l, *conf)) + cmd.AddCommand(replayStatusCommand(l, *conf)) + cmd.AddCommand(replayListCommand(l, *conf)) return cmd } diff --git a/cmd/replay_create.go b/cmd/replay_create.go index d891f8e308..35d7c26f49 100644 --- a/cmd/replay_create.go +++ b/cmd/replay_create.go @@ -19,7 +19,7 @@ import ( "github.com/odpf/optimus/core/set" ) -func replayCreateCommand(l log.Logger, conf config.Optimus) *cli.Command { +func replayCreateCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { var ( dryRun = false forceRun = false diff --git a/cmd/replay_list.go b/cmd/replay_list.go index d7a2dff2fb..4dc0a0e203 100644 --- a/cmd/replay_list.go +++ b/cmd/replay_list.go @@ -15,7 +15,7 @@ import ( "github.com/odpf/optimus/models" ) -func replayListCommand(l log.Logger, conf config.Optimus) *cli.Command { +func replayListCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { var ( projectName string reCmd = &cli.Command{ diff --git a/cmd/replay_status.go b/cmd/replay_status.go index 0045740374..bfc59eb28a 100644 --- a/cmd/replay_status.go +++ b/cmd/replay_status.go @@ -15,8 +15,10 @@ import ( "github.com/odpf/optimus/models" ) -func replayStatusCommand(l log.Logger, conf config.Optimus) *cli.Command { - var projectName string +func replayStatusCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { + var ( + projectName string + ) reCmd := &cli.Command{ Use: "status", diff --git a/cmd/resource.go b/cmd/resource.go index a402e48e44..574b48f0a4 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -23,7 +23,7 @@ import ( var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-zA-Z0-9_\-\.]+$`, `invalid name (can only contain characters A-Z (in either case), 0-9, "-", "_" or "." and must start with an alphanumeric character)`) -func resourceCommand(l log.Logger, conf config.Optimus, datastoreRepo models.DatastoreRepo, datastoreSpecsFs map[string]map[string]afero.Fs) *cli.Command { +func resourceCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Command { cmd := &cli.Command{ Use: "resource", Short: "Interact with data resource", @@ -31,11 +31,29 @@ func resourceCommand(l log.Logger, conf config.Optimus, datastoreRepo models.Dat "group:core": "true", }, } - cmd.AddCommand(createResourceSubCommand(l, conf, datastoreSpecsFs, datastoreRepo)) + + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + //init local specs + datastoreSpecFs := make(map[string]map[string]afero.Fs) + for _, namespace := range conf.Namespaces { + dtSpec := make(map[string]afero.Fs) + for _, dsConfig := range namespace.Datastore { + dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) + } + datastoreSpecFs[namespace.Name] = dtSpec + } + + cmd.AddCommand(createResourceSubCommand(l, *conf, datastoreSpecFs, datastoreRepo)) return cmd } -func createResourceSubCommand(l log.Logger, conf config.Optimus, datastoreSpecFs map[string]map[string]afero.Fs, datastoreRepo models.DatastoreRepo) *cli.Command { +func createResourceSubCommand(l log.Logger, conf config.ProjectConfig, datastoreSpecFs map[string]map[string]afero.Fs, datastoreRepo models.DatastoreRepo) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new resource", diff --git a/cmd/secret.go b/cmd/secret.go index b9374b2a15..be86380404 100644 --- a/cmd/secret.go +++ b/cmd/secret.go @@ -26,18 +26,26 @@ const ( secretTimeout = time.Minute * 2 ) -func secretCommand(l log.Logger, conf config.Optimus) *cli.Command { +func secretCommand(l log.Logger) *cli.Command { cmd := &cli.Command{ Use: "secret", Short: "Manage secrets to be used in jobs", } - cmd.AddCommand(secretSetSubCommand(l, conf)) - cmd.AddCommand(secretListSubCommand(l, conf)) - cmd.AddCommand(secretDeleteSubCommand(l, conf)) + + // TODO: find a way to load the config in one place + conf, err := config.LoadProjectConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + cmd.AddCommand(secretSetSubCommand(l, *conf)) + cmd.AddCommand(secretListSubCommand(l, *conf)) + cmd.AddCommand(secretDeleteSubCommand(l, *conf)) return cmd } -func secretSetSubCommand(l log.Logger, conf config.Optimus) *cli.Command { +func secretSetSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { var ( projectName string namespaceName string @@ -122,7 +130,7 @@ Use base64 flag if the value has been encoded. return secretCmd } -func secretListSubCommand(l log.Logger, conf config.Optimus) *cli.Command { +func secretListSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { var projectName string secretListCmd := &cli.Command{ @@ -142,7 +150,7 @@ func secretListSubCommand(l log.Logger, conf config.Optimus) *cli.Command { return secretListCmd } -func secretDeleteSubCommand(l log.Logger, conf config.Optimus) *cli.Command { +func secretDeleteSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { var projectName, namespaceName string cmd := &cli.Command{ diff --git a/cmd/serve.go b/cmd/serve.go index 44ca9eb5b1..a3457a5384 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -13,7 +13,7 @@ import ( "github.com/odpf/optimus/config" ) -func serveCommand(l log.Logger, conf config.Optimus) *cli.Command { +func serveCommand(l log.Logger) *cli.Command { c := &cli.Command{ Use: "serve", Short: "Starts optimus service", @@ -22,7 +22,14 @@ func serveCommand(l log.Logger, conf config.Optimus) *cli.Command { "group:other": "dev", }, RunE: func(c *cli.Command, args []string) error { - l.Info(coloredSuccess("Starting Optimus"), "version", config.Version) + // TODO: find a way to load the config in one place + conf, err := config.LoadServerConfig() + if err != nil { + l.Error(err.Error()) + return nil + } + + l.Info(coloredSuccess("Starting Optimus"), "version", config.BuildVersion) optimusServer, err := server.New(l, conf) defer optimusServer.Shutdown() if err != nil { @@ -36,5 +43,6 @@ func serveCommand(l log.Logger, conf config.Optimus) *cli.Command { return nil }, } + return c } diff --git a/cmd/server/server.go b/cmd/server/server.go index 56e6d6e49d..db8d763a2a 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -38,7 +38,7 @@ const ( BootstrapTimeout = time.Second * 10 ) -func checkRequiredConfigs(conf config.ServerConfig) error { +func checkRequiredConfigs(conf config.Serve) error { errRequiredMissing := errors.New("required config missing") if conf.IngressHost == "" { return fmt.Errorf("serve.ingress_host: %w", errRequiredMissing) diff --git a/cmd/version.go b/cmd/version.go index 81bff5a813..fe3433c954 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -37,6 +37,7 @@ func versionCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Comma // Print server version if isWithServer { + // TODO: find a way to load the config in one place conf, err := config.LoadProjectConfig() if err != nil { return err diff --git a/config/config_project.go b/config/config_project.go index 4261d2caef..a8515dedcb 100644 --- a/config/config_project.go +++ b/config/config_project.go @@ -1,5 +1,7 @@ package config +import "fmt" + type ProjectConfig struct { Version Version `mapstructure:"version"` Log LogConfig `mapstructure:"log"` @@ -31,3 +33,21 @@ type Namespace struct { Job Job `mapstructure:"job"` Datastore []Datastore `mapstructure:"datastore"` } + +func (c *ProjectConfig) GetNamespaceByName(name string) (*Namespace, error) { + if c.namespaceNameToNamespace == nil { + c.namespaceNameToNamespace = map[string]*Namespace{} + for _, namespace := range c.Namespaces { + if namespace == nil { + continue + } + c.namespaceNameToNamespace[namespace.Name] = namespace + } + } + + if c.namespaceNameToNamespace[name] == nil { + return nil, fmt.Errorf("namespace [%s] is not found", name) + } + + return c.namespaceNameToNamespace[name], nil +} diff --git a/ext/scheduler/airflow2/compiler/compiler.go b/ext/scheduler/airflow2/compiler/compiler.go index 6e49f83298..85ec5d7eb6 100644 --- a/ext/scheduler/airflow2/compiler/compiler.go +++ b/ext/scheduler/airflow2/compiler/compiler.go @@ -77,7 +77,7 @@ func (com *Compiler) Compile(schedulerTemplate []byte, namespaceSpec models.Name JobSpecDependencyTypeInter: string(models.JobSpecDependencyTypeInter), JobSpecDependencyTypeExtra: string(models.JobSpecDependencyTypeExtra), SLAMissDurationInSec: slaMissDurationInSec, - Version: config.Version, + Version: config.BuildVersion, Metadata: jobSpec.Metadata, }); err != nil { return models.Job{}, fmt.Errorf("failed to templatize job: %w", err) diff --git a/go.mod b/go.mod index f1ff638170..5cd0c48e61 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/emirpasic/gods v1.12.0 github.com/golang-migrate/migrate/v4 v4.14.1 github.com/golang/glog v1.0.0 // indirect - github.com/golang/protobuf v1.5.2 // indirect github.com/google/go-github v17.0.0+incompatible github.com/google/uuid v1.3.0 github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 @@ -63,6 +62,8 @@ require ( gorm.io/gorm v1.21.16 ) +require github.com/go-ozzo/ozzo-validation/v4 v4.3.0 + require ( cloud.google.com/go v0.94.0 // indirect cloud.google.com/go/storage v1.16.1 // indirect @@ -70,6 +71,7 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect + github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect @@ -79,6 +81,7 @@ require ( github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect github.com/google/btree v1.0.0 // indirect github.com/google/go-cmp v0.5.6 // indirect github.com/google/go-querystring v1.1.0 // indirect diff --git a/go.sum b/go.sum index 33fca38ef5..8ccd5b1a67 100644 --- a/go.sum +++ b/go.sum @@ -143,6 +143,8 @@ github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmV github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 h1:EFSB7Zo9Eg91v7MJPVsifUysc/wPdN+NOnVe6bWbdBM= github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.17.7/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= @@ -279,6 +281,8 @@ github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vb github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= From ac9f8005d43bf7cc18a3be2e72e3eb07c8da3168 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 12:32:51 +0700 Subject: [PATCH 11/54] refactor: naming project config to client config --- cmd/admin.go | 4 +- cmd/admin_build_instance.go | 2 +- cmd/backup.go | 2 +- cmd/backup_create.go | 2 +- cmd/backup_list.go | 2 +- cmd/backup_status.go | 2 +- cmd/config.go | 4 +- cmd/deploy.go | 14 +++--- cmd/job.go | 2 +- cmd/job_create.go | 2 +- cmd/job_hook.go | 2 +- cmd/job_render.go | 2 +- cmd/job_run.go | 2 +- cmd/job_validate.go | 2 +- cmd/namespace.go | 2 +- cmd/replay.go | 2 +- cmd/replay_create.go | 2 +- cmd/replay_list.go | 2 +- cmd/replay_status.go | 2 +- cmd/resource.go | 4 +- cmd/secret.go | 8 ++-- cmd/version.go | 2 +- .optimus.sample.yaml => config.sample.yaml | 45 +---------------- .../{config_project.go => config_client.go} | 4 +- config/loader.go | 12 ++--- config/loader_test.go | 28 +++++------ config/validation.go | 6 +-- config/validation_test.go | 4 +- optimus.sample.yaml | 48 +++++++++++++++++++ 29 files changed, 110 insertions(+), 105 deletions(-) rename .optimus.sample.yaml => config.sample.yaml (53%) rename config/{config_project.go => config_client.go} (93%) create mode 100644 optimus.sample.yaml diff --git a/cmd/admin.go b/cmd/admin.go index 8582184ee9..17b38663a9 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -16,7 +16,7 @@ func adminCommand(l log.Logger) *cli.Command { } // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil @@ -27,7 +27,7 @@ func adminCommand(l log.Logger) *cli.Command { } // adminBuildCommand builds a run instance -func adminBuildCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func adminBuildCommand(l log.Logger, conf config.ClientConfig) *cli.Command { cmd := &cli.Command{ Use: "build", Short: "Register a job run and get required assets", diff --git a/cmd/admin_build_instance.go b/cmd/admin_build_instance.go index 8b01712c3b..ccfcdd6338 100644 --- a/cmd/admin_build_instance.go +++ b/cmd/admin_build_instance.go @@ -27,7 +27,7 @@ const ( const unsubstitutedValue = "" -func adminBuildInstanceCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func adminBuildInstanceCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var ( optimusHost = conf.Host projectName = conf.Project.Name diff --git a/cmd/backup.go b/cmd/backup.go index e1a4aafb9e..679190a961 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -29,7 +29,7 @@ func backupCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Comman } // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil diff --git a/cmd/backup_create.go b/cmd/backup_create.go index a738093f2d..fb7a1cc0d1 100644 --- a/cmd/backup_create.go +++ b/cmd/backup_create.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupCreateCommand(l log.Logger, conf config.ProjectConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCreateCommand(l log.Logger, conf config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( backupCmd = &cli.Command{ Use: "create", diff --git a/cmd/backup_list.go b/cmd/backup_list.go index d851b4c43e..1779beb621 100644 --- a/cmd/backup_list.go +++ b/cmd/backup_list.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupListCommand(l log.Logger, conf config.ProjectConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupListCommand(l log.Logger, conf config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( backupCmd = &cli.Command{ Use: "list", diff --git a/cmd/backup_status.go b/cmd/backup_status.go index 9671c7dbd4..15b3443844 100644 --- a/cmd/backup_status.go +++ b/cmd/backup_status.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupStatusCommand(l log.Logger, conf config.ProjectConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupStatusCommand(l log.Logger, conf config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( project string backupCmd = &cli.Command{ diff --git a/cmd/config.go b/cmd/config.go index caad5c7187..9d9d832e2b 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -30,7 +30,7 @@ func configInitCommand(l log.Logger) *cli.Command { Use: "init", Short: "Initialize optimus configuration file", RunE: func(c *cli.Command, args []string) (err error) { - conf := config.ProjectConfig{ + conf := config.ClientConfig{ Version: config.Version(1), Host: defaultHost, } @@ -103,7 +103,7 @@ func configInitCommand(l log.Logger) *cli.Command { return c } -func projectConfigQuestions(conf config.ProjectConfig) (config.ProjectConfig, error) { +func projectConfigQuestions(conf config.ClientConfig) (config.ClientConfig, error) { conf.Project.Config = map[string]string{} registerMore := AnswerYes for registerMore == AnswerNo { diff --git a/cmd/deploy.go b/cmd/deploy.go index d8d24b964f..930c3fbb3d 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -46,7 +46,7 @@ func deployCommand(l log.Logger, pluginRepo models.PluginRepository, dsRepo mode ) // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil @@ -86,7 +86,7 @@ func deployCommand(l log.Logger, pluginRepo models.PluginRepository, dsRepo mode } // postDeploymentRequest send a deployment request to service -func postDeploymentRequest(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository, +func postDeploymentRequest(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository, datastoreRepo models.DatastoreRepo, datastoreSpecFs map[string]map[string]afero.Fs, namespaceNames []string, ignoreJobDeployment, ignoreResources, verbose bool) error { dialTimeoutCtx, dialCancel := context.WithTimeout(context.Background(), OptimusDialTimeout) @@ -146,7 +146,7 @@ func postDeploymentRequest(l log.Logger, conf config.ProjectConfig, pluginRepo m func deployAllJobs(deployTimeoutCtx context.Context, jobSpecificationServiceClient pb.JobSpecificationServiceClient, - l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, + l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository, datastoreRepo models.DatastoreRepo, namespaceNames []string, verbose bool, @@ -268,7 +268,7 @@ func deployAllJobs(deployTimeoutCtx context.Context, func deployAllResources(deployTimeoutCtx context.Context, resourceServiceClient pb.ResourceServiceClient, - l log.Logger, conf config.Optimus, pluginRepo models.PluginRepository, + l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository, datastoreRepo models.DatastoreRepo, datastoreSpecFs map[string]map[string]afero.Fs, namespaceNames []string, @@ -384,7 +384,7 @@ func deployAllResources(deployTimeoutCtx context.Context, func registerAllNamespaces( deployTimeoutCtx context.Context, namespaceServiceClient pb.NamespaceServiceClient, - l log.Logger, conf config.Optimus, namespaceNames []string, + l log.Logger, conf config.ClientConfig, namespaceNames []string, ) error { var selectedNamespaceNames []string if len(namespaceNames) > 0 { @@ -415,7 +415,7 @@ func registerAllNamespaces( } func registerNamespace(deployTimeoutCtx context.Context, namespaceServiceClient pb.NamespaceServiceClient, - l log.Logger, conf config.Optimus, namespaceName string, + l log.Logger, conf config.ClientConfig, namespaceName string, ) error { namespace, err := conf.GetNamespaceByName(namespaceName) if err != nil { @@ -443,7 +443,7 @@ func registerNamespace(deployTimeoutCtx context.Context, namespaceServiceClient func registerProject( deployTimeoutCtx context.Context, projectServiceClient pb.ProjectServiceClient, - l log.Logger, conf config.Optimus, + l log.Logger, conf config.ClientConfig, ) (err error) { projectSpec := &pb.ProjectSpecification{ Name: conf.Project.Name, diff --git a/cmd/job.go b/cmd/job.go index dd77ac7863..9e45434702 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -17,7 +17,7 @@ func jobCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command { } // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil diff --git a/cmd/job_create.go b/cmd/job_create.go index 40852c8650..01fe4fd16e 100644 --- a/cmd/job_create.go +++ b/cmd/job_create.go @@ -33,7 +33,7 @@ var ( specFileNames = []string{local.ResourceSpecFileName, local.JobSpecFileName} ) -func jobCreateCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobCreateCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new Job", diff --git a/cmd/job_hook.go b/cmd/job_hook.go index 074edb731e..bb450ec85f 100644 --- a/cmd/job_hook.go +++ b/cmd/job_hook.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobAddHookCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobAddHookCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "addhook", Aliases: []string{"add_hook", "add-hook", "addHook", "attach_hook", "attach-hook", "attachHook"}, diff --git a/cmd/job_render.go b/cmd/job_render.go index ad45b275a4..0d3df10cb2 100644 --- a/cmd/job_render.go +++ b/cmd/job_render.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobRenderTemplateCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobRenderTemplateCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "render", Short: "Apply template values in job specification to current 'render' directory", diff --git a/cmd/job_run.go b/cmd/job_run.go index b38e931e3f..e8fa6e31bb 100644 --- a/cmd/job_run.go +++ b/cmd/job_run.go @@ -22,7 +22,7 @@ const ( runJobTimeout = time.Minute * 1 ) -func jobRunCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobRunCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { var ( namespaceName string projectName = conf.Project.Name diff --git a/cmd/job_validate.go b/cmd/job_validate.go index 06d1be1fbe..ee0ea4525e 100644 --- a/cmd/job_validate.go +++ b/cmd/job_validate.go @@ -23,7 +23,7 @@ const ( validateTimeout = time.Minute * 5 ) -func jobValidateCommand(l log.Logger, conf config.ProjectConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobValidateCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { var ( verbose bool namespaceName string diff --git a/cmd/namespace.go b/cmd/namespace.go index 592fb7720c..1d88c15d9b 100644 --- a/cmd/namespace.go +++ b/cmd/namespace.go @@ -9,7 +9,7 @@ import ( "github.com/odpf/optimus/config" ) -func askToSelectNamespace(l log.Logger, conf config.ProjectConfig) (*config.Namespace, error) { +func askToSelectNamespace(l log.Logger, conf config.ClientConfig) (*config.Namespace, error) { options := make([]string, len(conf.Namespaces)) if len(conf.Namespaces) == 0 { return nil, errors.New("no namespace found in config file") diff --git a/cmd/replay.go b/cmd/replay.go index 02cb7e7497..c545eba8ff 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -63,7 +63,7 @@ func replayCommand(l log.Logger) *cli.Command { } // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil diff --git a/cmd/replay_create.go b/cmd/replay_create.go index 35d7c26f49..aa35930dd6 100644 --- a/cmd/replay_create.go +++ b/cmd/replay_create.go @@ -19,7 +19,7 @@ import ( "github.com/odpf/optimus/core/set" ) -func replayCreateCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func replayCreateCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var ( dryRun = false forceRun = false diff --git a/cmd/replay_list.go b/cmd/replay_list.go index 4dc0a0e203..93669d5194 100644 --- a/cmd/replay_list.go +++ b/cmd/replay_list.go @@ -15,7 +15,7 @@ import ( "github.com/odpf/optimus/models" ) -func replayListCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func replayListCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var ( projectName string reCmd = &cli.Command{ diff --git a/cmd/replay_status.go b/cmd/replay_status.go index bfc59eb28a..12a7f77b23 100644 --- a/cmd/replay_status.go +++ b/cmd/replay_status.go @@ -15,7 +15,7 @@ import ( "github.com/odpf/optimus/models" ) -func replayStatusCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func replayStatusCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var ( projectName string ) diff --git a/cmd/resource.go b/cmd/resource.go index 574b48f0a4..cdc40a9bca 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -33,7 +33,7 @@ func resourceCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Comm } // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil @@ -53,7 +53,7 @@ func resourceCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Comm return cmd } -func createResourceSubCommand(l log.Logger, conf config.ProjectConfig, datastoreSpecFs map[string]map[string]afero.Fs, datastoreRepo models.DatastoreRepo) *cli.Command { +func createResourceSubCommand(l log.Logger, conf config.ClientConfig, datastoreSpecFs map[string]map[string]afero.Fs, datastoreRepo models.DatastoreRepo) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new resource", diff --git a/cmd/secret.go b/cmd/secret.go index be86380404..8cfa48256b 100644 --- a/cmd/secret.go +++ b/cmd/secret.go @@ -33,7 +33,7 @@ func secretCommand(l log.Logger) *cli.Command { } // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { l.Error(err.Error()) return nil @@ -45,7 +45,7 @@ func secretCommand(l log.Logger) *cli.Command { return cmd } -func secretSetSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func secretSetSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var ( projectName string namespaceName string @@ -130,7 +130,7 @@ Use base64 flag if the value has been encoded. return secretCmd } -func secretListSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func secretListSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var projectName string secretListCmd := &cli.Command{ @@ -150,7 +150,7 @@ func secretListSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command return secretListCmd } -func secretDeleteSubCommand(l log.Logger, conf config.ProjectConfig) *cli.Command { +func secretDeleteSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { var projectName, namespaceName string cmd := &cli.Command{ diff --git a/cmd/version.go b/cmd/version.go index fe3433c954..f145fc118f 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -38,7 +38,7 @@ func versionCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Comma // Print server version if isWithServer { // TODO: find a way to load the config in one place - conf, err := config.LoadProjectConfig() + conf, err := config.LoadClientConfig() if err != nil { return err } diff --git a/.optimus.sample.yaml b/config.sample.yaml similarity index 53% rename from .optimus.sample.yaml rename to config.sample.yaml index c0d3e03789..b1bf6bb6b1 100644 --- a/.optimus.sample.yaml +++ b/config.sample.yaml @@ -1,16 +1,10 @@ version: 1 -######################################## -# COMMON CONFIG (APPLIED FOR SERVER & CLIENT) -######################################## # logging configuration log: # debug, info, warning, error, fatal - default 'info' level: info - - - ######################################## # SERVER CONFIG ######################################## @@ -56,41 +50,4 @@ log: # profile_addr: ":9110" # # # jaeger collector address to send application traces -# jaeger_addr: "http://localhost:14268/api/traces" - - - - -######################################## -# PROJECT CONFIG -######################################## - -# used to connect optimus service -#host: localhost:9100 - -# for configuring optimus project -#project: -# name: sample_project -# # project variables usable in specifications -# config: -# environment: integration -# scheduler_host: http://example.io/ -# # storage_path is used for storing compiled job specifications that can be -# # consumed by schedulers like Airflow -# # it supports multiple schemes like: file://, gcs:// -# storage_path: file://absolute_path_to_a_directory - -# for configuring optimus namespace -#namespace: -# name: sample_namespace -# job: -# # relative path pointing to folder where job specifications are stored -# path: "job" -# datastore: -# # optimus is capable of supporting multiple datastores -# type: bigquery -# # relative path where resource spec for BQ are stored -# path: "bq" -# # namespace variables usable in specifications -# config: {} - +# jaeger_addr: "http://localhost:14268/api/traces" \ No newline at end of file diff --git a/config/config_project.go b/config/config_client.go similarity index 93% rename from config/config_project.go rename to config/config_client.go index a8515dedcb..f21165ebc2 100644 --- a/config/config_project.go +++ b/config/config_client.go @@ -2,7 +2,7 @@ package config import "fmt" -type ProjectConfig struct { +type ClientConfig struct { Version Version `mapstructure:"version"` Log LogConfig `mapstructure:"log"` Host string `mapstructure:"host"` // optimus server host @@ -34,7 +34,7 @@ type Namespace struct { Datastore []Datastore `mapstructure:"datastore"` } -func (c *ProjectConfig) GetNamespaceByName(name string) (*Namespace, error) { +func (c *ClientConfig) GetNamespaceByName(name string) (*Namespace, error) { if c.namespaceNameToNamespace == nil { c.namespaceNameToNamespace = map[string]*Namespace{} for _, namespace := range c.Namespaces { diff --git a/config/loader.go b/config/loader.go index 4a83c29d08..3bc312da93 100644 --- a/config/loader.go +++ b/config/loader.go @@ -12,7 +12,7 @@ import ( const ( ErrFailedToRead = "unable to read optimus config file %v (%s)" - DefaultFilename = ".optimus" + DefaultFilename = "optimus" DefaultFileExtension = "yaml" DefaultEnvPrefix = "OPTIMUS" ) @@ -46,11 +46,11 @@ func init() { homePath = p } -// LoadProjectConfig load the project specific config from these locations: -// 1. env var. eg. OPTIMUS_PROJECT, OPTIMUS_NAMESPACES, etc +// LoadClientConfig load the project specific config from these locations: +// 1. env var. eg. OPTIMUS_PROJECT, OPTIMUS_NAMESPACES, etc TODO: skip this part // 2. filepath. ./optimus -c "path/to/config/optimus.yaml" // 3. current dir. Optimus will look at current directory if there's optimus.yaml there, use it -func LoadProjectConfig(filepath ...string) (*ProjectConfig, error) { +func LoadClientConfig(filepath ...string) (*ClientConfig, error) { fs := afero.NewReadOnlyFs(afero.NewOsFs()) return loadProjectConfigFs(fs, filepath...) } @@ -65,8 +65,8 @@ func LoadServerConfig(filepath ...string) (*ServerConfig, error) { return loadServerConfigFs(fs, filepath...) } -func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ProjectConfig, error) { - cfg := &ProjectConfig{} +func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ClientConfig, error) { + cfg := &ClientConfig{} if len(filepath) == 0 { filepath = append(filepath, "") diff --git a/config/loader_test.go b/config/loader_test.go index 2d21da8139..33b212063a 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -63,8 +63,8 @@ type ConfigTestSuite struct { execPath string homePath string - expectedProjectConfig *ProjectConfig - expectedServerConfig *ServerConfig + expectedClientConfig *ClientConfig + expectedServerConfig *ServerConfig } func (s *ConfigTestSuite) SetupTest() { @@ -100,14 +100,14 @@ func TestConfig(t *testing.T) { suite.Run(t, new(ConfigTestSuite)) } -func (s *ConfigTestSuite) TestInternal_LoadProjectConfigFs() { +func (s *ConfigTestSuite) TestLoadClientConfigFs() { s.a.WriteFile(path.Join(s.currPath, filename+"."+fileExtension), []byte(projectConfig), fs.ModeTemporary) s.Run("WhenFilepathIsEmpty", func() { p, err := loadProjectConfigFs(s.a.Fs) s.Assert().NoError(err) s.Assert().NotNil(p) - s.Assert().Equal(s.expectedProjectConfig, p) + s.Assert().Equal(s.expectedClientConfig, p) }) s.Run("WhenFilepathIsExist", func() { @@ -134,7 +134,7 @@ func (s *ConfigTestSuite) TestInternal_LoadProjectConfigFs() { }) } -func (s *ConfigTestSuite) TestInternal_LoadServerConfigFs() { +func (s *ConfigTestSuite) TestLoadServerConfigFs() { s.a.WriteFile(path.Join(s.execPath, filename+"."+fileExtension), []byte(serverConfig), fs.ModeTemporary) s.a.WriteFile(path.Join(s.homePath, filename+"."+fileExtension), []byte(`version: 3`), fs.ModeTemporary) @@ -178,10 +178,10 @@ func (s *ConfigTestSuite) TestLoadProjectConfig() { defer os.Remove(fpath) os.WriteFile(fpath, []byte(projectConfig), 0400) - conf, err := LoadProjectConfig(fpath) - s.Assert().NoError(err) + conf, err := LoadClientConfig(fpath) + s.NoError(err) s.Assert().NotNil(conf) - s.Assert().Equal(s.expectedProjectConfig, conf) + s.Assert().Equal(s.expectedClientConfig, conf) } func (s *ConfigTestSuite) TestLoadServerConfig() { @@ -197,12 +197,12 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { } func (s *ConfigTestSuite) initExpectedProjectConfig() { - s.expectedProjectConfig = &ProjectConfig{} - s.expectedProjectConfig.Version = Version(1) - s.expectedProjectConfig.Log = LogConfig{Level: "info"} + s.expectedClientConfig = &ClientConfig{} + s.expectedClientConfig.Version = Version(1) + s.expectedClientConfig.Log = LogConfig{Level: "info"} - s.expectedProjectConfig.Host = "localhost:9100" - s.expectedProjectConfig.Project = Project{ + s.expectedClientConfig.Host = "localhost:9100" + s.expectedClientConfig.Project = Project{ Name: "sample_project", Config: map[string]string{ "environment": "integration", @@ -223,7 +223,7 @@ func (s *ConfigTestSuite) initExpectedProjectConfig() { Path: "./jobs-b", }, }) - s.expectedProjectConfig.Namespaces = namespaces + s.expectedClientConfig.Namespaces = namespaces } func (s *ConfigTestSuite) initExpectedServerConfig() { diff --git a/config/validation.go b/config/validation.go index 80362a7248..506c79d41a 100644 --- a/config/validation.go +++ b/config/validation.go @@ -11,15 +11,15 @@ import ( // Validate validate the config as an input. If not valid, it returns error func Validate(conf interface{}) error { switch c := conf.(type) { - case ProjectConfig: - return validateProjectConfig(c) + case ClientConfig: + return validateClientConfig(c) case ServerConfig: return validateServerConfig(c) } return errors.New("error") } -func validateProjectConfig(conf ProjectConfig) error { +func validateClientConfig(conf ClientConfig) error { // implement this return validation.ValidateStruct(&conf, validation.Field(&conf.Version, validation.Required), diff --git a/config/validation_test.go b/config/validation_test.go index 13e3c569ff..ea9e69e6e0 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -8,7 +8,7 @@ import ( type ValidationTestSuite struct { suite.Suite - defaultProjectConfig ProjectConfig + defaultProjectConfig ClientConfig } func (s *ValidationTestSuite) SetupTest() { @@ -86,7 +86,7 @@ func (s *ValidationTestSuite) TestValidate_Fail() { } func (s *ValidationTestSuite) initDefaultProjectConfig() { - s.defaultProjectConfig = ProjectConfig{} + s.defaultProjectConfig = ClientConfig{} s.defaultProjectConfig.Version = Version(1) s.defaultProjectConfig.Log = LogConfig{Level: "info"} diff --git a/optimus.sample.yaml b/optimus.sample.yaml new file mode 100644 index 0000000000..25d12f852a --- /dev/null +++ b/optimus.sample.yaml @@ -0,0 +1,48 @@ +######################################## +# PROJECT CONFIG +######################################## + +version: 1 + +# logging configuration +log: + # debug, info, warning, error, fatal - default 'info' + level: info + +# used to connect optimus service +#host: localhost:9100 + +# for configuring optimus project +#project: +# name: sample_project +# # project variables usable in specifications +# config: +# environment: integration +# scheduler_host: http://example.io/ +# # storage_path is used for storing compiled job specifications that can be +# # consumed by schedulers like Airflow +# # it supports multiple schemes like: file://, gcs:// +# storage_path: file://absolute_path_to_a_directory + +# for configuring optimus namespaces +#namespaces: +#- name: sample_namespace +# job: +# # relative path pointing to folder where job specifications are stored +# path: "ns1/job" +# datastore: +# # optimus is capable of supporting multiple datastores +# type: bigquery +# # relative path where resource spec for BQ are stored +# path: "bq" +# # namespace variables usable in specifications +# config: {} +#- name: sample_namespace_2 +# job: +# path: "ns2/job" +# datastore: +# type: bigquery +# path: "bq" +# config: {} + + From 08e57dfaedfacabb0cf8649f7cf13cdaa8bd9056 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 12:44:09 +0700 Subject: [PATCH 12/54] refactor: initliazie jsonLogger and telemtry on serve comand --- cmd/commands.go | 3 ++- cmd/serve.go | 44 +++++++++++++++++++++++++++++++++++++++----- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/cmd/commands.go b/cmd/commands.go index 51ed3a1131..4a4882febd 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -114,12 +114,13 @@ func New(plainLog log.Logger, jsonLog log.Logger, pluginRepo models.PluginReposi cmd.AddCommand(jobCommand(plainLog, pluginRepo)) cmd.AddCommand(deployCommand(plainLog, pluginRepo, dsRepo)) cmd.AddCommand(resourceCommand(plainLog, dsRepo)) - cmd.AddCommand(serveCommand(jsonLog)) cmd.AddCommand(replayCommand(plainLog)) cmd.AddCommand(backupCommand(plainLog, dsRepo)) cmd.AddCommand(adminCommand(plainLog)) cmd.AddCommand(secretCommand(plainLog)) + cmd.AddCommand(serveCommand()) + addExtensionCommand(cmd, plainLog) return cmd } diff --git a/cmd/serve.go b/cmd/serve.go index a3457a5384..1e558379c1 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,16 +4,19 @@ import ( "fmt" "os" "os/signal" + "strings" "syscall" - "github.com/odpf/salt/log" - cli "github.com/spf13/cobra" - + "github.com/hashicorp/go-hclog" + hPlugin "github.com/hashicorp/go-plugin" "github.com/odpf/optimus/cmd/server" "github.com/odpf/optimus/config" + "github.com/odpf/optimus/plugin" + "github.com/odpf/salt/log" + cli "github.com/spf13/cobra" ) -func serveCommand(l log.Logger) *cli.Command { +func serveCommand() *cli.Command { c := &cli.Command{ Use: "serve", Short: "Starts optimus service", @@ -25,10 +28,41 @@ func serveCommand(l log.Logger) *cli.Command { // TODO: find a way to load the config in one place conf, err := config.LoadServerConfig() if err != nil { - l.Error(err.Error()) return nil } + // initiate jsonLogger + var jsonLogger log.Logger + pluginLogLevel := hclog.Info + if conf.Log.Level != "" { + jsonLogger = log.NewLogrus(log.LogrusWithLevel(conf.Log.Level), log.LogrusWithWriter(os.Stderr)) + if strings.ToLower(conf.Log.Level) == "debug" { + pluginLogLevel = hclog.Debug + } + } else { + jsonLogger = log.NewLogrus(log.LogrusWithLevel("INFO"), log.LogrusWithWriter(os.Stderr)) + } + + // discover and load plugins. TODO: refactor this + if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ + Name: "optimus", + Output: os.Stdout, + Level: pluginLogLevel, + })); err != nil { + hPlugin.CleanupClients() + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + // Make sure we clean up any managed plugins at the end of this + defer hPlugin.CleanupClients() + + // init telemetry + teleShutdown, err := config.InitTelemetry(jsonLogger, conf.Telemetry) + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + defer teleShutdown() l.Info(coloredSuccess("Starting Optimus"), "version", config.BuildVersion) optimusServer, err := server.New(l, conf) defer optimusServer.Shutdown() From e591abaefb33d7e2a4cafbe461c4722d23232323 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 15:12:53 +0700 Subject: [PATCH 13/54] refactor: define log level as const --- config/config_common.go | 13 +++++++++++++ config/validation.go | 9 ++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/config/config_common.go b/config/config_common.go index c31724fa66..216ba55057 100644 --- a/config/config_common.go +++ b/config/config_common.go @@ -4,9 +4,22 @@ import "strconv" // Contains shared config for server and client (project) +// Config is just an alias for interface{} +type Config interface{} + // Version implement fmt.Stringer type Version int +type LogLevel string + +const ( + LogLevelDebug LogLevel = "DEBUG" + LogLevelInfo = "INFO" + LogLevelWarning = "INFO" + LogLevelError = "ERROR" + LogLevelFatal = "FATAL" +) + type LogConfig struct { Level string `mapstructure:"level" default:"info"` // log level - debug, info, warning, error, fatal Format string `mapstructure:"format"` // format strategy - plain, json diff --git a/config/validation.go b/config/validation.go index 506c79d41a..99d277eb10 100644 --- a/config/validation.go +++ b/config/validation.go @@ -9,7 +9,7 @@ import ( ) // Validate validate the config as an input. If not valid, it returns error -func Validate(conf interface{}) error { +func Validate(conf Config) error { switch c := conf.(type) { case ClientConfig: return validateClientConfig(c) @@ -24,6 +24,13 @@ func validateClientConfig(conf ClientConfig) error { return validation.ValidateStruct(&conf, validation.Field(&conf.Version, validation.Required), validation.Field(&conf.Host, validation.Required), + validation.Field(&conf.Log.Level, validation.In( + LogLevelDebug, + LogLevelInfo, + LogLevelWarning, + LogLevelError, + LogLevelFatal, + )), validation.Field(&conf.Namespaces, validation.By(validateNamespaces)), // ... etc ) From f6569f4871fe49d3496791c72a722928b45e939e Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 15:41:14 +0700 Subject: [PATCH 14/54] refactor: logger for command serve --- cmd/config.go | 3 +-- cmd/logger.go | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/serve.go | 14 ++----------- 3 files changed, 58 insertions(+), 14 deletions(-) create mode 100644 cmd/logger.go diff --git a/cmd/config.go b/cmd/config.go index 9d9d832e2b..42fc07496a 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -5,11 +5,10 @@ import ( "io/ioutil" "github.com/AlecAivazis/survey/v2" + "github.com/odpf/optimus/config" "github.com/odpf/salt/log" cli "github.com/spf13/cobra" "gopkg.in/yaml.v2" - - "github.com/odpf/optimus/config" ) const ( diff --git a/cmd/logger.go b/cmd/logger.go new file mode 100644 index 0000000000..c88a5e4bdf --- /dev/null +++ b/cmd/logger.go @@ -0,0 +1,55 @@ +package cmd + +import ( + "fmt" + "os" + + "github.com/odpf/optimus/config" + "github.com/odpf/salt/log" + "github.com/sirupsen/logrus" +) + +// conf -> client / server / ,, , ,, +// logger -> json / plain / xml / ... +// func -> 4 + +type loggerType int + +const ( + jsonLogger loggerType = iota + plainLogger +) + +type plainFormatter int + +func (p *plainFormatter) Format(entry *logrus.Entry) ([]byte, error) { + if len(entry.Data) > 0 { + var data string + for key, val := range entry.Data { + data += fmt.Sprintf("%s: %v ", key, val) + } + return []byte(fmt.Sprintf("%s %s\n", entry.Message, data)), nil + } + return []byte(fmt.Sprintf("%s\n", entry.Message)), nil +} + +func initLogger(t loggerType, conf config.LogConfig) log.Logger { + if conf.Level == "" { + conf.Level = config.LogLevelInfo + } + + switch t { + case jsonLogger: + return log.NewLogrus( + log.LogrusWithLevel(conf.Level), + log.LogrusWithWriter(os.Stderr), + ) + case plainLogger: + return log.NewLogrus( + log.LogrusWithLevel(conf.Level), + log.LogrusWithFormatter(new(plainFormatter)), + ) + } + + return nil +} diff --git a/cmd/serve.go b/cmd/serve.go index 1e558379c1..2341e3847d 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "os/signal" - "strings" "syscall" "github.com/hashicorp/go-hclog" @@ -12,7 +11,6 @@ import ( "github.com/odpf/optimus/cmd/server" "github.com/odpf/optimus/config" "github.com/odpf/optimus/plugin" - "github.com/odpf/salt/log" cli "github.com/spf13/cobra" ) @@ -32,16 +30,8 @@ func serveCommand() *cli.Command { } // initiate jsonLogger - var jsonLogger log.Logger + l := initLogger(jsonLogger, conf.Log) pluginLogLevel := hclog.Info - if conf.Log.Level != "" { - jsonLogger = log.NewLogrus(log.LogrusWithLevel(conf.Log.Level), log.LogrusWithWriter(os.Stderr)) - if strings.ToLower(conf.Log.Level) == "debug" { - pluginLogLevel = hclog.Debug - } - } else { - jsonLogger = log.NewLogrus(log.LogrusWithLevel("INFO"), log.LogrusWithWriter(os.Stderr)) - } // discover and load plugins. TODO: refactor this if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ @@ -57,7 +47,7 @@ func serveCommand() *cli.Command { defer hPlugin.CleanupClients() // init telemetry - teleShutdown, err := config.InitTelemetry(jsonLogger, conf.Telemetry) + teleShutdown, err := config.InitTelemetry(l, conf.Telemetry) if err != nil { fmt.Printf("ERROR: %s\n", err.Error()) os.Exit(1) From 6a108a3cd90910fd58b31b29e18b159fb64755a3 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 23 Mar 2022 15:45:15 +0700 Subject: [PATCH 15/54] refactor: remove unused params on cmd.New --- cmd/commands.go | 3 +- main.go | 74 ++----------------------------------------------- 2 files changed, 4 insertions(+), 73 deletions(-) diff --git a/cmd/commands.go b/cmd/commands.go index 4a4882febd..eb22037f80 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -15,7 +15,6 @@ import ( "github.com/mattn/go-isatty" "github.com/odpf/optimus/models" "github.com/odpf/salt/cmdx" - "github.com/odpf/salt/log" "github.com/odpf/salt/term" cli "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" @@ -59,7 +58,7 @@ type JobSpecRepository interface { // default output of logging should go to stdout // interactive output like progress bars should go to stderr // unless the stdout/err is a tty, colors/progressbar should be disabled -func New(plainLog log.Logger, jsonLog log.Logger, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { +func New(pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { disableColoredOut = !isTerminal(os.Stdout) cmd := &cli.Command{ diff --git a/main.go b/main.go index 94986df728..b6b08e4c3d 100644 --- a/main.go +++ b/main.go @@ -1,96 +1,28 @@ package main import ( - "errors" "fmt" "math/rand" + _ "net/http/pprof" "os" - "strings" "time" - "github.com/hashicorp/go-hclog" - hPlugin "github.com/hashicorp/go-plugin" - "github.com/odpf/salt/log" - "github.com/sirupsen/logrus" - "github.com/odpf/optimus/cmd" - "github.com/odpf/optimus/config" _ "github.com/odpf/optimus/ext/datastore" "github.com/odpf/optimus/models" - "github.com/odpf/optimus/plugin" ) -var errRequestFail = errors.New("🔥 unable to complete request successfully") - -type PlainFormatter struct{} - -func (p *PlainFormatter) Format(entry *logrus.Entry) ([]byte, error) { - if len(entry.Data) > 0 { - var data string - for key, val := range entry.Data { - data += fmt.Sprintf("%s: %v ", key, val) - } - return []byte(fmt.Sprintf("%s %s\n", entry.Message, data)), nil - } - return []byte(fmt.Sprintf("%s\n", entry.Message)), nil -} - //nolint:forbidigo func main() { rand.Seed(time.Now().UTC().UnixNano()) - optimusConfig, err := config.LoadOptimusConfig() - if err != nil { - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(1) - } - - var jsonLogger log.Logger - var plainLogger log.Logger - pluginLogLevel := hclog.Info - if optimusConfig.Log.Level != "" { - jsonLogger = log.NewLogrus(log.LogrusWithLevel(optimusConfig.Log.Level), log.LogrusWithWriter(os.Stderr)) - plainLogger = log.NewLogrus(log.LogrusWithLevel(optimusConfig.Log.Level), log.LogrusWithFormatter(new(PlainFormatter))) - if strings.ToLower(optimusConfig.Log.Level) == "debug" { - pluginLogLevel = hclog.Debug - } - } else { - jsonLogger = log.NewLogrus(log.LogrusWithLevel("INFO"), log.LogrusWithWriter(os.Stderr)) - plainLogger = log.NewLogrus(log.LogrusWithLevel("INFO"), log.LogrusWithFormatter(new(PlainFormatter))) - } - - // init telemetry - teleShutdown, err := config.InitTelemetry(jsonLogger, optimusConfig.Telemetry) - if err != nil { - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(1) - } - - // discover and load plugins - if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ - Name: "optimus", - Output: os.Stdout, - Level: pluginLogLevel, - })); err != nil { - hPlugin.CleanupClients() - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(1) - } - command := cmd.New( - plainLogger, - jsonLogger, - *optimusConfig, models.PluginRegistry, models.DatastoreRegistry, ) + if err := command.Execute(); err != nil { - hPlugin.CleanupClients() - teleShutdown() - // no need to print err here, `command` does that already - fmt.Println(errRequestFail) + fmt.Println(err) os.Exit(1) } - hPlugin.CleanupClients() - teleShutdown() } From 8cf97a1fcf6a9ed0030f6110043cf92a3f0918b7 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 11:44:26 +0700 Subject: [PATCH 16/54] refactor: use exported function instead of internal --- config/loader.go | 118 +++++++++---------- config/loader_test.go | 259 ++++++++++++++++++++++++++---------------- 2 files changed, 220 insertions(+), 157 deletions(-) diff --git a/config/loader.go b/config/loader.go index 3bc312da93..78f4d7e689 100644 --- a/config/loader.go +++ b/config/loader.go @@ -1,7 +1,7 @@ package config import ( - "errors" + "fmt" "os" "github.com/odpf/salt/config" @@ -15,15 +15,14 @@ const ( DefaultFilename = "optimus" DefaultFileExtension = "yaml" DefaultEnvPrefix = "OPTIMUS" + EmptyPath = "" ) var ( - filename = DefaultFilename - fileExtension = DefaultFileExtension // ASK: are we providing file extension other than yaml? - envPrefix = DefaultEnvPrefix - currPath string - execPath string - homePath string + FS = afero.NewReadOnlyFs(afero.NewOsFs()) + currPath string + execPath string + homePath string ) func init() { @@ -47,85 +46,80 @@ func init() { } // LoadClientConfig load the project specific config from these locations: -// 1. env var. eg. OPTIMUS_PROJECT, OPTIMUS_NAMESPACES, etc TODO: skip this part -// 2. filepath. ./optimus -c "path/to/config/optimus.yaml" -// 3. current dir. Optimus will look at current directory if there's optimus.yaml there, use it -func LoadClientConfig(filepath ...string) (*ClientConfig, error) { - fs := afero.NewReadOnlyFs(afero.NewOsFs()) - return loadProjectConfigFs(fs, filepath...) -} - -// LoadServerConfig load the server specific config from these locations: -// 1. env var. eg. OPTIMUS_SERVE_PORT, etc -// 2. filepath. ./optimus -c "path/to/config.yaml" -// 3. executable binary location -// 4. home dir -func LoadServerConfig(filepath ...string) (*ServerConfig, error) { - fs := afero.NewReadOnlyFs(afero.NewOsFs()) - return loadServerConfigFs(fs, filepath...) -} - -func loadProjectConfigFs(fs afero.Fs, filepath ...string) (*ClientConfig, error) { +// 1. filepath. ./optimus -c "path/to/config/optimus.yaml" +// 2. current dir. Optimus will look at current directory if there's optimus.yaml there, use it +func LoadClientConfig(filePath string) (*ClientConfig, error) { cfg := &ClientConfig{} - if len(filepath) == 0 { - filepath = append(filepath, "") - } + // getViperWithDefault + SetFs + v := viper.New() + v.SetFs(FS) - err := loadConfig(cfg, fs, filepath[0], currPath) - if err != nil { - return nil, err + opts := []config.LoaderOption{ + config.WithViper(v), + config.WithName(DefaultFilename), + config.WithType(DefaultFileExtension), } - return cfg, nil -} - -func loadServerConfigFs(fs afero.Fs, filepath ...string) (*ServerConfig, error) { - cfg := &ServerConfig{} - - if len(filepath) == 0 { - filepath = append(filepath, "") + // load opt from filepath if exist + if filePath != EmptyPath { + if err := validateFilepath(FS, filePath); err != nil { + return nil, err // if filepath not valid, returns err + } + opts = append(opts, config.WithFile(filePath)) + } else { + // load opt from current directory + opts = append(opts, config.WithPath(currPath)) } - err := loadConfig(cfg, fs, filepath[0], execPath, homePath) - if err != nil { + // load the config + l := config.NewLoader(opts...) + if err := l.Load(cfg); err != nil { return nil, err } return cfg, nil } -func loadConfig(cfg interface{}, fs afero.Fs, paths ...string) error { +// LoadServerConfig load the server specific config from these locations: +// 1. filepath. ./optimus -c "path/to/config.yaml" +// 2. env var. eg. OPTIMUS_SERVE_PORT, etc +// 3. executable binary location +// 4. home dir +func LoadServerConfig(filePath string) (*ServerConfig, error) { + cfg := &ServerConfig{} + // getViperWithDefault + SetFs v := viper.New() - v.SetFs(fs) + v.SetFs(FS) opts := []config.LoaderOption{ config.WithViper(v), - config.WithName(filename), - config.WithType(fileExtension), - config.WithEnvPrefix(envPrefix), - config.WithEnvKeyReplacer(".", "_"), + config.WithName(DefaultFilename), + config.WithType(DefaultFileExtension), } - if len(paths) > 0 { - fpath := paths[0] - dirPaths := paths[1:] - - if fpath != "" { - if err := validateFilepath(fs, fpath); err != nil { - return err - } - opts = append(opts, config.WithFile(fpath)) + // load opt from filepath if exist + if filePath != EmptyPath { + if err := validateFilepath(FS, filePath); err != nil { + return nil, err // if filepath not valid, returns err } + opts = append(opts, config.WithFile(filePath)) + } else { + // load opt from env var + opts = append(opts, config.WithEnvPrefix(DefaultEnvPrefix), config.WithEnvKeyReplacer(".", "_")) - for _, path := range dirPaths { - opts = append(opts, config.WithPath(path)) - } + // load opt from exec & home directory + opts = append(opts, config.WithPath(execPath), config.WithPath(homePath)) } + // load the config l := config.NewLoader(opts...) - return l.Load(cfg) + if err := l.Load(cfg); err != nil { + return nil, err + } + + return cfg, nil } func validateFilepath(fs afero.Fs, fpath string) error { @@ -134,7 +128,7 @@ func validateFilepath(fs afero.Fs, fpath string) error { return err } if !f.Mode().IsRegular() { - return errors.New("not a file") + return fmt.Errorf("%s not a file", fpath) } return nil } diff --git a/config/loader_test.go b/config/loader_test.go index 33b212063a..3b57f074b2 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -1,14 +1,14 @@ -package config +package config_test import ( "io/fs" "os" "path" - "path/filepath" "strings" "testing" "time" + "github.com/odpf/optimus/config" "github.com/spf13/afero" "github.com/stretchr/testify/suite" ) @@ -63,8 +63,8 @@ type ConfigTestSuite struct { execPath string homePath string - expectedClientConfig *ClientConfig - expectedServerConfig *ServerConfig + expectedClientConfig *config.ClientConfig + expectedServerConfig *config.ServerConfig } func (s *ConfigTestSuite) SetupTest() { @@ -86,6 +86,8 @@ func (s *ConfigTestSuite) SetupTest() { s.homePath = p s.a.Fs.MkdirAll(s.homePath, fs.ModeTemporary) + config.FS = s.a.Fs + s.initExpectedProjectConfig() s.initExpectedServerConfig() } @@ -100,109 +102,137 @@ func TestConfig(t *testing.T) { suite.Run(t, new(ConfigTestSuite)) } -func (s *ConfigTestSuite) TestLoadClientConfigFs() { - s.a.WriteFile(path.Join(s.currPath, filename+"."+fileExtension), []byte(projectConfig), fs.ModeTemporary) +func (s *ConfigTestSuite) TestLoadClientConfig() { + currFilePath := path.Join(s.currPath, config.DefaultFilename+"."+config.DefaultFileExtension) + s.a.WriteFile(currFilePath, []byte(projectConfig), fs.ModeTemporary) s.Run("WhenFilepathIsEmpty", func() { - p, err := loadProjectConfigFs(s.a.Fs) - s.Assert().NoError(err) - s.Assert().NotNil(p) - s.Assert().Equal(s.expectedClientConfig, p) + s.Run("WhenConfigInCurrentPathIsExist", func() { + conf, err := config.LoadClientConfig(config.EmptyPath) + + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal(s.expectedClientConfig, conf) + }) + + s.Run("WhenConfigInCurrentPathNotExist", func() { + s.a.Remove(currFilePath) + defer s.a.WriteFile(currFilePath, []byte(projectConfig), fs.ModeTemporary) + + conf, err := config.LoadClientConfig(config.EmptyPath) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + }) }) s.Run("WhenFilepathIsExist", func() { - samplePath := "./sample/path/config.yaml" - b := strings.Builder{} - b.WriteString(projectConfig) - b.WriteString(`- name: namespace-c + s.Run("WhenFilePathIsvalid", func() { + samplePath := "./sample/path/config.yaml" + b := strings.Builder{} + b.WriteString(projectConfig) + b.WriteString(`- name: namespace-c job: path: ./jobs-c `) - s.a.WriteFile(samplePath, []byte(b.String()), fs.ModeTemporary) - defer s.a.Fs.RemoveAll(samplePath) + s.a.WriteFile(samplePath, []byte(b.String()), fs.ModeTemporary) + defer s.a.Fs.RemoveAll(samplePath) - p, err := loadProjectConfigFs(s.a.Fs, samplePath) - s.Assert().NoError(err) - s.Assert().NotNil(p) - s.Assert().Len(p.Namespaces, 3) - }) + conf, err := config.LoadClientConfig(samplePath) + + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Len(conf.Namespaces, 3) + }) - s.Run("WhenLoadConfigIsFailed", func() { - p, err := loadProjectConfigFs(s.a.Fs, "/path/not/exist") - s.Assert().Error(err) - s.Assert().Nil(p) + s.Run("WhenFilePathIsNotValid", func() { + conf, err := config.LoadClientConfig("/path/not/exist") + + s.Assert().Error(err) + s.Assert().Nil(conf) + }) }) + } -func (s *ConfigTestSuite) TestLoadServerConfigFs() { - s.a.WriteFile(path.Join(s.execPath, filename+"."+fileExtension), []byte(serverConfig), fs.ModeTemporary) - s.a.WriteFile(path.Join(s.homePath, filename+"."+fileExtension), []byte(`version: 3`), fs.ModeTemporary) +func (s *ConfigTestSuite) TestLoadServerConfig() { + execFilePath := path.Join(s.execPath, config.DefaultFilename+"."+config.DefaultFileExtension) + homeFilePath := path.Join(s.homePath, config.DefaultFilename+"."+config.DefaultFilename) + s.a.WriteFile(execFilePath, []byte(serverConfig), fs.ModeTemporary) + s.a.WriteFile(homeFilePath, []byte(`version: 3`), fs.ModeTemporary) + s.initServerConfigEnv() s.Run("WhenFilepathIsEmpty", func() { - conf, err := loadServerConfigFs(s.a.Fs) - s.Assert().NoError(err) - s.Assert().NotNil(conf) - s.Assert().Equal(s.expectedServerConfig, conf) // should load from exec path - - s.a.Remove(path.Join(s.execPath, filename+"."+fileExtension)) - defer s.a.WriteFile(path.Join(s.execPath, filename+"."+fileExtension), []byte(serverConfig), fs.ModeTemporary) - conf, err = loadServerConfigFs(s.a.Fs) - s.Assert().NoError(err) - s.Assert().NotNil(conf) - s.Assert().Equal("3", conf.Version.String()) // should load from home dir + s.Run("WhenEnvExist", func() { + conf, err := config.LoadServerConfig(config.EmptyPath) + + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal("4", conf.Version.String()) // should load from env var + }) + + s.Run("WhenEnvNotExist", func() { + s.unsetServerConfigEnv() + defer s.initServerConfigEnv() + + conf, err := config.LoadServerConfig(config.EmptyPath) + + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal(s.expectedServerConfig, conf) // should load from exec dir + }) + + s.Run("WhenEnvNotExistAndExecDirNotExist", func() { + s.unsetServerConfigEnv() + s.a.Remove(execFilePath) + defer s.initServerConfigEnv() + defer s.a.WriteFile(execFilePath, []byte(serverConfig), fs.ModeTemporary) + + conf, err := config.LoadServerConfig(config.EmptyPath) + + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal("3", conf.Version.String()) // should load from home dir + }) + + s.Run("WhenConfigNotFound", func() { + s.a.Remove(execFilePath) + s.a.Remove(homeFilePath) + defer s.a.WriteFile(execFilePath, []byte(serverConfig), fs.ModeTemporary) + defer s.a.WriteFile(homeFilePath, []byte(`version: 3`), fs.ModeTemporary) + + conf, err := config.LoadServerConfig(config.EmptyPath) + s.Assert().NoError(err) + s.Assert().NotNil(conf) + }) }) s.Run("WhenFilepathIsExist", func() { - samplePath := "./sample/path/config.yaml" - s.a.WriteFile(samplePath, []byte("version: 2"), os.ModeTemporary) - - conf, err := loadServerConfigFs(s.a.Fs, samplePath) - s.Assert().NoError(err) - s.Assert().NotNil(conf) - s.Assert().Equal("2", conf.Version.String()) - }) - - s.Run("WhenLoadConfigIsFailed", func() { - dirPath := "./sample/path" - s.a.Fs.MkdirAll(dirPath, os.ModeTemporary) - - conf, err := loadServerConfigFs(s.a.Fs, dirPath) - s.Assert().Error(err) - s.Assert().Nil(conf) + s.Run("WhenFilePathIsValid", func() { + samplePath := "./sample/path/config.yaml" + s.a.WriteFile(samplePath, []byte("version: 2"), os.ModeTemporary) + + conf, err := config.LoadServerConfig(samplePath) + + s.Assert().NoError(err) + s.Assert().NotNil(conf) + s.Assert().Equal("2", conf.Version.String()) + }) + + s.Run("WhenFilePathIsNotValid", func() { + conf, err := config.LoadServerConfig("/path/not/exist") + s.Assert().Error(err) + s.Assert().Nil(conf) + }) }) } -func (s *ConfigTestSuite) TestLoadProjectConfig() { - path := os.TempDir() - fpath := filepath.Join(path, filename+"."+fileExtension) - defer os.Remove(fpath) - os.WriteFile(fpath, []byte(projectConfig), 0400) - - conf, err := LoadClientConfig(fpath) - s.NoError(err) - s.Assert().NotNil(conf) - s.Assert().Equal(s.expectedClientConfig, conf) -} - -func (s *ConfigTestSuite) TestLoadServerConfig() { - path := os.TempDir() - fpath := filepath.Join(path, filename+"."+fileExtension) - defer os.Remove(fpath) - os.WriteFile(fpath, []byte(serverConfig), 0400) - - conf, err := LoadServerConfig(fpath) - s.Assert().NoError(err) - s.Assert().NotNil(conf) - s.Assert().Equal(s.expectedServerConfig, conf) -} - func (s *ConfigTestSuite) initExpectedProjectConfig() { - s.expectedClientConfig = &ClientConfig{} - s.expectedClientConfig.Version = Version(1) - s.expectedClientConfig.Log = LogConfig{Level: "info"} + s.expectedClientConfig = &config.ClientConfig{} + s.expectedClientConfig.Version = config.Version(1) + s.expectedClientConfig.Log = config.LogConfig{Level: "info"} s.expectedClientConfig.Host = "localhost:9100" - s.expectedClientConfig.Project = Project{ + s.expectedClientConfig.Project = config.Project{ Name: "sample_project", Config: map[string]string{ "environment": "integration", @@ -210,16 +240,16 @@ func (s *ConfigTestSuite) initExpectedProjectConfig() { "storage_path": "file://absolute_path_to_a_directory", }, } - namespaces := []*Namespace{} - namespaces = append(namespaces, &Namespace{ + namespaces := []*config.Namespace{} + namespaces = append(namespaces, &config.Namespace{ Name: "namespace-a", - Job: Job{ + Job: config.Job{ Path: "./jobs-a", }, }) - namespaces = append(namespaces, &Namespace{ + namespaces = append(namespaces, &config.Namespace{ Name: "namespace-b", - Job: Job{ + Job: config.Job{ Path: "./jobs-b", }, }) @@ -227,11 +257,11 @@ func (s *ConfigTestSuite) initExpectedProjectConfig() { } func (s *ConfigTestSuite) initExpectedServerConfig() { - s.expectedServerConfig = &ServerConfig{} - s.expectedServerConfig.Version = Version(1) - s.expectedServerConfig.Log = LogConfig{Level: "info"} + s.expectedServerConfig = &config.ServerConfig{} + s.expectedServerConfig.Version = config.Version(1) + s.expectedServerConfig.Log = config.LogConfig{Level: "info"} - s.expectedServerConfig.Serve = Serve{} + s.expectedServerConfig.Serve = config.Serve{} s.expectedServerConfig.Serve.Port = 9100 s.expectedServerConfig.Serve.Host = "localhost" s.expectedServerConfig.Serve.IngressHost = "optimus.example.io:80" @@ -239,16 +269,55 @@ func (s *ConfigTestSuite) initExpectedServerConfig() { s.expectedServerConfig.Serve.ReplayNumWorkers = 1 s.expectedServerConfig.Serve.ReplayWorkerTimeout = 100 * time.Second s.expectedServerConfig.Serve.ReplayRunTimeout = 10 * time.Second - s.expectedServerConfig.Serve.DB = DBConfig{} + s.expectedServerConfig.Serve.DB = config.DBConfig{} s.expectedServerConfig.Serve.DB.DSN = "postgres://user:password@localhost:5432/database?sslmode=disable" s.expectedServerConfig.Serve.DB.MaxIdleConnection = 5 s.expectedServerConfig.Serve.DB.MaxOpenConnection = 10 - s.expectedServerConfig.Scheduler = SchedulerConfig{} + s.expectedServerConfig.Scheduler = config.SchedulerConfig{} s.expectedServerConfig.Scheduler.Name = "airflow2" s.expectedServerConfig.Scheduler.SkipInit = true - s.expectedServerConfig.Telemetry = TelemetryConfig{} + s.expectedServerConfig.Telemetry = config.TelemetryConfig{} s.expectedServerConfig.Telemetry.ProfileAddr = ":9110" s.expectedServerConfig.Telemetry.JaegerAddr = "http://localhost:14268/api/traces" } + +func (s *ConfigTestSuite) initServerConfigEnv() { + os.Setenv("OPTIMUS_VERSION", "4") + os.Setenv("OPTIMUS_LOG_LEVEL", "info") + os.Setenv("OPTIMUS_SERVE_PORT", "9100") + os.Setenv("OPTIMUS_SERVE_HOST", "localhost") + os.Setenv("OPTIMUS_SERVE_INGRESS_HOST", "optimus.example.io:80") + os.Setenv("OPTIMUS_SERVE_APP_KEY", "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc") + os.Setenv("OPTIMUS_SERVE_REPLAY_NUM_WORKERS", "1") + os.Setenv("OPTIMUS_SERVE_REPLAY_WORKER_TIMEOUT", "100s") + os.Setenv("OPTIMUS_SERVE_REPLAY_RUN_TIMEOUT", "10s") + os.Setenv("OPTIMUS_SERVE_DB_DSN", "postgres://user:password@localhost:5432/database?sslmode=disable") + os.Setenv("OPTIMUS_SERVE_DB_MAX_IDLE_CONNECTION", "5") + os.Setenv("OPTIMUS_SERVE_DB_MAX_OPEN_CONNECTION", "10") + os.Setenv("OPTIMUS_SCHEDULER_NAME", "airflow2") + os.Setenv("OPTIMUS_SCHEDULER_SKIP_INIT", "true") + os.Setenv("OPTIMUS_TELEMETRY_PROFILE_ADDR", ":9110") + os.Setenv("OPTIMUS_TELEMETRY_JAEGER_ADDR", "http://localhost:14268/api/traces") +} + +func (s *ConfigTestSuite) unsetServerConfigEnv() { + os.Unsetenv("OPTIMUS_VERSION") + os.Unsetenv("OPTIMUS_LOG_LEVEL") + os.Unsetenv("OPTIMUS_SERVE_PORT") + os.Unsetenv("OPTIMUS_SERVE_HOST") + os.Unsetenv("OPTIMUS_SERVE_INGRESS_HOST") + os.Unsetenv("OPTIMUS_SERVE_APP_KEY") + os.Unsetenv("OPTIMUS_SERVE_REPLAY_NUM_WORKERS") + os.Unsetenv("OPTIMUS_SERVE_REPLAY_WORKER_TIMEOUT") + os.Unsetenv("OPTIMUS_SERVE_REPLAY_RUN_TIMEOUT") + os.Unsetenv("OPTIMUS_SERVE_DB_DSN") + os.Unsetenv("OPTIMUS_SERVE_DB_MAX_IDLE_CONNECTION") + os.Unsetenv("OPTIMUS_SERVE_DB_MAX_OPEN_CONNECTION") + os.Unsetenv("OPTIMUS_SCHEDULER_NAME") + os.Unsetenv("OPTIMUS_SCHEDULER_SKIP_INIT") + os.Unsetenv("OPTIMUS_TELEMETRY_PROFILE_ADDR") + os.Unsetenv("OPTIMUS_TELEMETRY_JAEGER_ADDR") + +} From ccf1f0f6706e1529f2d456d69251fe1400caa3bb Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 12:21:54 +0700 Subject: [PATCH 17/54] revert: use server implementation from main branch --- cmd/serve.go | 1 + cmd/server/optimus.go | 24 +++++++------- cmd/server/scheduler.go | 6 ++-- config/config.go | 44 ++++++++++++++++++++++++++ config/config_server.go | 4 --- config/loader.go | 69 +++++++++++++++++++++++++++++++++++++++++ 6 files changed, 129 insertions(+), 19 deletions(-) create mode 100644 config/config.go diff --git a/cmd/serve.go b/cmd/serve.go index 2341e3847d..582268b915 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -26,6 +26,7 @@ func serveCommand() *cli.Command { // TODO: find a way to load the config in one place conf, err := config.LoadServerConfig() if err != nil { + panic(err.Error()) return nil } diff --git a/cmd/server/optimus.go b/cmd/server/optimus.go index 699b4103bd..f0855cf04e 100644 --- a/cmd/server/optimus.go +++ b/cmd/server/optimus.go @@ -29,7 +29,7 @@ import ( type setupFn func() error type OptimusServer struct { - conf config.Optimus + conf config.ServerConfig logger log.Logger appKey models.ApplicationKey @@ -42,15 +42,15 @@ type OptimusServer struct { cleanupFn []func() } -func New(l log.Logger, conf config.Optimus) (*OptimusServer, error) { - addr := fmt.Sprintf("%s:%d", conf.Server.Host, conf.Server.Port) +func New(l log.Logger, conf config.ServerConfig) (*OptimusServer, error) { + addr := fmt.Sprintf("%s:%d", conf.Serve.Host, conf.Serve.Port) server := &OptimusServer{ conf: conf, logger: l, serverAddr: addr, } - if err := checkRequiredConfigs(conf.Server); err != nil { + if err := checkRequiredConfigs(conf.Serve); err != nil { return server, err } @@ -76,7 +76,7 @@ func New(l log.Logger, conf config.Optimus) (*OptimusServer, error) { func (s *OptimusServer) setupAppKey() error { var err error - s.appKey, err = models.NewApplicationSecret(s.conf.Server.AppKey) + s.appKey, err = models.NewApplicationSecret(s.conf.Serve.AppKey) if err != nil { return fmt.Errorf("NewApplicationSecret: %w", err) } @@ -85,12 +85,12 @@ func (s *OptimusServer) setupAppKey() error { func (s *OptimusServer) setupDB() error { var err error - if err := postgres.Migrate(s.conf.Server.DB.DSN); err != nil { + if err := postgres.Migrate(s.conf.Serve.DB.DSN); err != nil { return fmt.Errorf("postgres.Migrate: %w", err) } // TODO: Connect should accept DBConfig - s.dbConn, err = postgres.Connect(s.conf.Server.DB.DSN, s.conf.Server.DB.MaxIdleConnection, - s.conf.Server.DB.MaxOpenConnection, s.logger.Writer()) + s.dbConn, err = postgres.Connect(s.conf.Serve.DB.DSN, s.conf.Serve.DB.MaxIdleConnection, + s.conf.Serve.DB.MaxOpenConnection, s.logger.Writer()) if err != nil { return fmt.Errorf("postgres.Connect: %w", err) } @@ -215,9 +215,9 @@ func (s *OptimusServer) setupHandlers() error { ) replayManager := job.NewManager(s.logger, replayWorkerFactory, replaySpecRepoFac, utils.NewUUIDProvider(), job.ReplayManagerConfig{ - NumWorkers: s.conf.Server.ReplayNumWorkers, - WorkerTimeout: s.conf.Server.ReplayWorkerTimeout, - RunTimeout: s.conf.Server.ReplayRunTimeout, + NumWorkers: s.conf.Serve.ReplayNumWorkers, + WorkerTimeout: s.conf.Serve.ReplayWorkerTimeout, + RunTimeout: s.conf.Serve.ReplayRunTimeout, }, scheduler, replayValidator, replaySyncer) notificationContext, cancelNotifiers := context.WithCancel(context.Background()) @@ -326,7 +326,7 @@ func (s *OptimusServer) setupHandlers() error { // runtime service instance over grpc pb.RegisterRuntimeServiceServer(s.grpcServer, v1handler.NewRuntimeServiceServer( s.logger, - config.Version, + config.BuildVersion, jobService, eventService, namespaceService, diff --git a/cmd/server/scheduler.go b/cmd/server/scheduler.go index bd20c37b88..1f38cc22db 100644 --- a/cmd/server/scheduler.go +++ b/cmd/server/scheduler.go @@ -19,8 +19,8 @@ import ( "github.com/odpf/optimus/utils" ) -func initScheduler(l log.Logger, conf config.Optimus, projectRepoFac *projectRepoFactory) (models.SchedulerUnit, error) { - jobCompiler := compiler.NewCompiler(conf.Server.IngressHost) +func initScheduler(l log.Logger, conf config.ServerConfig, projectRepoFac *projectRepoFactory) (models.SchedulerUnit, error) { + jobCompiler := compiler.NewCompiler(conf.Serve.IngressHost) // init default scheduler var scheduler models.SchedulerUnit switch conf.Scheduler.Name { @@ -55,7 +55,7 @@ func initScheduler(l log.Logger, conf config.Optimus, projectRepoFac *projectRep return scheduler, nil } -func initPrimeCluster(l log.Logger, conf config.Optimus, jobrunRepoFac *jobRunRepoFactory, dbConn *gorm.DB) (context.CancelFunc, error) { +func initPrimeCluster(l log.Logger, conf config.ServerConfig, jobrunRepoFac *jobRunRepoFactory, dbConn *gorm.DB) (context.CancelFunc, error) { models.ManualScheduler = prime.NewScheduler( // careful global variable jobrunRepoFac, func() time.Time { diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000000..c4a4c9725e --- /dev/null +++ b/config/config.go @@ -0,0 +1,44 @@ +package config + +import ( + "fmt" + "strconv" +) + +const ( + KeyServeReplayNumWorkers = "serve.replay_num_workers" +) + +type Optimus struct { + // configuration version + Version int `mapstructure:"version"` + // optimus server host + Host string `mapstructure:"host"` + + Project Project `mapstructure:"project"` + Namespaces []*Namespace `mapstructure:"namespaces"` + + Server Serve `mapstructure:"serve"` + Log LogConfig `mapstructure:"log"` + Scheduler SchedulerConfig `mapstructure:"scheduler"` + Telemetry TelemetryConfig `mapstructure:"telemetry"` + + namespaceNameToNamespace map[string]*Namespace +} + +func (o *Optimus) GetNamespaceByName(name string) (*Namespace, error) { + if o.namespaceNameToNamespace == nil { + o.namespaceNameToNamespace = make(map[string]*Namespace) + for _, namespace := range o.Namespaces { + o.namespaceNameToNamespace[namespace.Name] = namespace + } + } + if o.namespaceNameToNamespace[name] == nil { + return nil, fmt.Errorf("namespace [%s] is not found", name) + } + return o.namespaceNameToNamespace[name], nil +} + +func (o *Optimus) GetVersion() string { + return strconv.Itoa(o.Version) +} diff --git a/config/config_server.go b/config/config_server.go index e3940cae35..0e05d2c934 100644 --- a/config/config_server.go +++ b/config/config_server.go @@ -4,10 +4,6 @@ import ( "time" ) -const ( - KeyServeReplayNumWorkers = "serve.replay_num_workers" -) - type ServerConfig struct { Version Version `mapstructure:"version"` Log LogConfig `mapstructure:"log"` diff --git a/config/loader.go b/config/loader.go index 78f4d7e689..a7a84f21c6 100644 --- a/config/loader.go +++ b/config/loader.go @@ -1,8 +1,10 @@ package config import ( + "errors" "fmt" "os" + "strings" "github.com/odpf/salt/config" "github.com/spf13/afero" @@ -45,6 +47,73 @@ func init() { homePath = p } +// LoadOptimusConfig Load configuration file from following paths (LEGACY) +// ./ +// / +// ~/.optimus/ +// Namespaces will be loaded only from current project ./ +func LoadOptimusConfig(dirPaths ...string) (*Optimus, error) { + fs := afero.NewReadOnlyFs(afero.NewOsFs()) + + var targetPath string + if len(dirPaths) > 0 { + targetPath = dirPaths[0] + } else { + currPath, err := os.Getwd() + if err != nil { + return nil, fmt.Errorf("error getting current work directory path: %w", err) + } + targetPath = currPath + } + + optimus := Optimus{} + if err := loadConfig(&optimus, fs, targetPath); err != nil { + return nil, errors.New("error loading config") + } + if err := validateNamespaceDuplication(&optimus); err != nil { + return nil, err + } + return &optimus, nil +} + +func validateNamespaceDuplication(optimus *Optimus) error { + nameToAppearance := make(map[string]int) + for _, namespace := range optimus.Namespaces { + nameToAppearance[namespace.Name]++ + } + var duplicateNames []string + for name, appearance := range nameToAppearance { + if appearance > 1 { + duplicateNames = append(duplicateNames, name) + } + } + if len(duplicateNames) > 0 { + return fmt.Errorf("namespaces [%s] are duplicate", strings.Join(duplicateNames, ", ")) + } + return nil +} + +func loadConfig(cfg interface{}, fs afero.Fs, dirPath string) error { + // getViperWithDefault + SetFs + v := viper.New() + v.SetConfigName("config") + v.SetConfigType("yaml") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) + v.SetFs(fs) + + opts := []config.LoaderOption{ + config.WithViper(v), + config.WithName(DefaultFilename), + config.WithType(DefaultFileExtension), + config.WithEnvPrefix("OPTIMUS"), + config.WithEnvKeyReplacer(".", "_"), + config.WithPath(dirPath), + } + + l := config.NewLoader(opts...) + return l.Load(cfg) +} + // LoadClientConfig load the project specific config from these locations: // 1. filepath. ./optimus -c "path/to/config/optimus.yaml" // 2. current dir. Optimus will look at current directory if there's optimus.yaml there, use it From 099e55b84eb3640216f076ee8357b481a6009a25 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 16:00:38 +0700 Subject: [PATCH 18/54] refactor: initiate client config & logger in each commands --- cmd/admin.go | 23 +++-- cmd/admin_build_instance.go | 2 +- cmd/backup.go | 25 +++-- cmd/backup_create.go | 2 +- cmd/backup_list.go | 2 +- cmd/backup_status.go | 2 +- cmd/commands.go | 20 ++-- cmd/config.go | 9 +- cmd/deploy.go | 60 ++++++----- cmd/extension.go | 20 ++-- cmd/job.go | 33 ++++-- cmd/job_create.go | 2 +- cmd/job_hook.go | 2 +- cmd/job_render.go | 2 +- cmd/job_run.go | 2 +- cmd/job_validate.go | 2 +- cmd/logger.go | 16 ++- cmd/namespace.go | 2 +- cmd/replay.go | 27 +++-- cmd/replay_create.go | 2 +- cmd/replay_list.go | 2 +- cmd/replay_status.go | 2 +- cmd/resource.go | 199 +++++++++++++++++++----------------- cmd/secret.go | 31 ++++-- cmd/serve.go | 5 +- cmd/version.go | 33 +++--- 26 files changed, 303 insertions(+), 224 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index 17b38663a9..d415cd8a6f 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -8,26 +8,35 @@ import ( ) // adminCommand registers internal administration commands -func adminCommand(l log.Logger) *cli.Command { +func adminCommand() *cli.Command { + var configFilePath string + var conf = &config.ClientConfig{} + var l log.Logger = initLogger(plainLoggerType, conf.Log) + cmd := &cli.Command{ Use: "admin", Short: "Internal administration commands", Hidden: true, } + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + var err error + + conf, err = config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + l = initLogger(plainLoggerType, conf.Log) - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) return nil } - cmd.AddCommand(adminBuildCommand(l, *conf)) + cmd.AddCommand(adminBuildCommand(l, conf)) return cmd } // adminBuildCommand builds a run instance -func adminBuildCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func adminBuildCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { cmd := &cli.Command{ Use: "build", Short: "Register a job run and get required assets", diff --git a/cmd/admin_build_instance.go b/cmd/admin_build_instance.go index ccfcdd6338..d51c27e7ff 100644 --- a/cmd/admin_build_instance.go +++ b/cmd/admin_build_instance.go @@ -27,7 +27,7 @@ const ( const unsubstitutedValue = "" -func adminBuildInstanceCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func adminBuildInstanceCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( optimusHost = conf.Host projectName = conf.Project.Name diff --git a/cmd/backup.go b/cmd/backup.go index 679190a961..2ae103f1ee 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -15,7 +15,11 @@ const ( backupTimeout = time.Minute * 15 ) -func backupCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCommand(datastoreRepo models.DatastoreRepo) *cli.Command { + var configFilePath string + var conf = &config.ClientConfig{} + var l log.Logger = initLogger(plainLoggerType, conf.Log) + cmd := &cli.Command{ Use: "backup", Short: "Backup a resource and its downstream", @@ -27,16 +31,21 @@ func backupCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Comman "group:core": "true", }, } + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + var err error + + conf, err = config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + l = initLogger(plainLoggerType, conf.Log) - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) return nil } - cmd.AddCommand(backupCreateCommand(l, *conf, datastoreRepo)) - cmd.AddCommand(backupListCommand(l, *conf, datastoreRepo)) - cmd.AddCommand(backupStatusCommand(l, *conf, datastoreRepo)) + cmd.AddCommand(backupCreateCommand(l, conf, datastoreRepo)) + cmd.AddCommand(backupListCommand(l, conf, datastoreRepo)) + cmd.AddCommand(backupStatusCommand(l, conf, datastoreRepo)) return cmd } diff --git a/cmd/backup_create.go b/cmd/backup_create.go index fb7a1cc0d1..e8f9e4ae89 100644 --- a/cmd/backup_create.go +++ b/cmd/backup_create.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupCreateCommand(l log.Logger, conf config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCreateCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( backupCmd = &cli.Command{ Use: "create", diff --git a/cmd/backup_list.go b/cmd/backup_list.go index 1779beb621..e7396acb98 100644 --- a/cmd/backup_list.go +++ b/cmd/backup_list.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupListCommand(l log.Logger, conf config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupListCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( backupCmd = &cli.Command{ Use: "list", diff --git a/cmd/backup_status.go b/cmd/backup_status.go index 15b3443844..b70ada9dac 100644 --- a/cmd/backup_status.go +++ b/cmd/backup_status.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupStatusCommand(l log.Logger, conf config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupStatusCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { var ( project string backupCmd = &cli.Command{ diff --git a/cmd/commands.go b/cmd/commands.go index eb22037f80..fc6aa20a02 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -108,19 +108,19 @@ func New(pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.C cmdx.SetHelp(cmd) cmd.PersistentFlags().BoolVar(&disableColoredOut, "no-color", disableColoredOut, "Disable colored output") - cmd.AddCommand(versionCommand(plainLog, pluginRepo)) - cmd.AddCommand(configCommand(plainLog)) - cmd.AddCommand(jobCommand(plainLog, pluginRepo)) - cmd.AddCommand(deployCommand(plainLog, pluginRepo, dsRepo)) - cmd.AddCommand(resourceCommand(plainLog, dsRepo)) - cmd.AddCommand(replayCommand(plainLog)) - cmd.AddCommand(backupCommand(plainLog, dsRepo)) - cmd.AddCommand(adminCommand(plainLog)) - cmd.AddCommand(secretCommand(plainLog)) + cmd.AddCommand(versionCommand(pluginRepo)) + cmd.AddCommand(configCommand()) + cmd.AddCommand(jobCommand(pluginRepo)) + cmd.AddCommand(deployCommand(pluginRepo, dsRepo)) + cmd.AddCommand(resourceCommand(dsRepo)) + cmd.AddCommand(replayCommand()) + cmd.AddCommand(backupCommand(dsRepo)) + cmd.AddCommand(adminCommand()) + cmd.AddCommand(secretCommand()) cmd.AddCommand(serveCommand()) - addExtensionCommand(cmd, plainLog) + addExtensionCommand(cmd) return cmd } diff --git a/cmd/config.go b/cmd/config.go index 42fc07496a..7818d9a389 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -6,7 +6,6 @@ import ( "github.com/AlecAivazis/survey/v2" "github.com/odpf/optimus/config" - "github.com/odpf/salt/log" cli "github.com/spf13/cobra" "gopkg.in/yaml.v2" ) @@ -15,16 +14,16 @@ const ( defaultHost = "localhost" ) -func configCommand(l log.Logger) *cli.Command { +func configCommand() *cli.Command { c := &cli.Command{ Use: "config", Short: "Manage optimus configuration required to deploy specifications", } - c.AddCommand(configInitCommand(l)) + c.AddCommand(configInitCommand()) return c } -func configInitCommand(l log.Logger) *cli.Command { +func configInitCommand() *cli.Command { c := &cli.Command{ Use: "init", Short: "Initialize optimus configuration file", @@ -95,6 +94,8 @@ func configInitCommand(l log.Logger) *cli.Command { if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension), confMarshaled, 0655); err != nil { return err } + + l := initLogger(plainLoggerType, conf.Log) l.Info(coloredSuccess("Configuration initialised successfully")) return nil }, diff --git a/cmd/deploy.go b/cmd/deploy.go index 930c3fbb3d..81d42ed2ca 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -27,57 +27,63 @@ const ( ) // deployCommand pushes current repo to optimus service -func deployCommand(l log.Logger, pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { +func deployCommand(pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { var ( namespaces []string ignoreJobs bool ignoreResources bool verbose bool - cmd = &cli.Command{ - Use: "deploy", - Short: "Deploy current optimus project to server", - Long: heredoc.Doc(`Apply local changes to destination server which includes creating/updating/deleting - jobs and creating/updating datastore resources`), - Example: "optimus deploy [--ignore-resources|--ignore-jobs]", - Annotations: map[string]string{ - "group:core": "true", - }, - } + configFilePath string ) - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) - return nil - } - - //init local specs - datastoreSpecFs := make(map[string]map[string]afero.Fs) - for _, namespace := range conf.Namespaces { - dtSpec := make(map[string]afero.Fs) - for _, dsConfig := range namespace.Datastore { - dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) - } - datastoreSpecFs[namespace.Name] = dtSpec + cmd := &cli.Command{ + Use: "deploy", + Short: "Deploy current optimus project to server", + Long: heredoc.Doc(`Apply local changes to destination server which includes creating/updating/deleting + jobs and creating/updating datastore resources`), + Example: "optimus deploy [--ignore-resources|--ignore-jobs]", + Annotations: map[string]string{ + "group:core": "true", + }, } + cmd.Flags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") cmd.Flags().StringSliceVarP(&namespaces, "namespaces", "N", nil, "Selected namespaces of optimus project") cmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Print details related to deployment stages") cmd.Flags().BoolVar(&ignoreJobs, "ignore-jobs", false, "Ignore deployment of jobs") cmd.Flags().BoolVar(&ignoreResources, "ignore-resources", false, "Ignore deployment of resources") cmd.RunE = func(c *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + conf, err := config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + + // init logger + l := initLogger(plainLoggerType, conf.Log) + + //init local specs + datastoreSpecFs := make(map[string]map[string]afero.Fs) + for _, namespace := range conf.Namespaces { + dtSpec := make(map[string]afero.Fs) + for _, dsConfig := range namespace.Datastore { + dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) + } + datastoreSpecFs[namespace.Name] = dtSpec + } l.Info(fmt.Sprintf("Deploying project: %s to %s", conf.Project.Name, conf.Host)) start := time.Now() if err := validateNamespaces(datastoreSpecFs, namespaces); err != nil { return err } - err := postDeploymentRequest(l, *conf, pluginRepo, dsRepo, datastoreSpecFs, namespaces, ignoreJobs, ignoreResources, verbose) + + err = postDeploymentRequest(l, *conf, pluginRepo, dsRepo, datastoreSpecFs, namespaces, ignoreJobs, ignoreResources, verbose) if err != nil { return err } + l.Info(coloredSuccess("\nDeployment completed, took %s", time.Since(start).Round(time.Second))) return nil } diff --git a/cmd/extension.go b/cmd/extension.go index 46db4dfba2..804dfb87e7 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -8,13 +8,13 @@ import ( "strings" "github.com/google/go-github/github" + "github.com/odpf/optimus/config" + "github.com/odpf/optimus/extension" "github.com/odpf/salt/log" cli "github.com/spf13/cobra" - - "github.com/odpf/optimus/extension" ) -func addExtensionCommand(cmd *cli.Command, l log.Logger) { +func addExtensionCommand(cmd *cli.Command) { ctx := context.Background() httpClient := &http.Client{} githubClient := github.NewClient(nil) @@ -35,25 +35,29 @@ func addExtensionCommand(cmd *cli.Command, l log.Logger) { reservedCommands..., ) - cmd.AddCommand(extensionCommand(ctx, extension, l)) + cmd.AddCommand(extensionCommand(ctx, extension)) commands := generateCommands(manifest, extension.Run) for _, c := range commands { cmd.AddCommand(c) } } -func extensionCommand(ctx context.Context, extension *extension.Extension, l log.Logger) *cli.Command { +func extensionCommand(ctx context.Context, extension *extension.Extension) *cli.Command { c := &cli.Command{ Use: "extension SUBCOMMAND", Aliases: []string{"ext"}, Short: "Operate with extension", } - c.AddCommand(extensionInstallCommand(ctx, extension, l)) + c.AddCommand(extensionInstallCommand(ctx, extension)) return c } -func extensionInstallCommand(ctx context.Context, installer extension.Installer, l log.Logger) *cli.Command { - var alias string +func extensionInstallCommand(ctx context.Context, installer extension.Installer) *cli.Command { + var ( + alias string + l log.Logger = initLogger(plainLoggerType, config.LogConfig{}) + ) + installCmd := &cli.Command{ Use: "install OWNER/REPO", Short: "Install an extension", diff --git a/cmd/job.go b/cmd/job.go index 9e45434702..814b28f99e 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -7,8 +7,13 @@ import ( cli "github.com/spf13/cobra" ) -func jobCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command { +func jobCommand(pluginRepo models.PluginRepository) *cli.Command { + var configFilePath string + var conf = &config.ClientConfig{} + var l log.Logger = initLogger(plainLoggerType, conf.Log) + cmd := &cli.Command{ + Use: "job", Short: "Interact with schedulable Job", Annotations: map[string]string{ @@ -16,18 +21,26 @@ func jobCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command { }, } - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + var err error + + conf, err = config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + l = initLogger(plainLoggerType, conf.Log) + return nil } - cmd.AddCommand(jobCreateCommand(l, *conf, pluginRepo)) - cmd.AddCommand(jobAddHookCommand(l, *conf, pluginRepo)) - cmd.AddCommand(jobRenderTemplateCommand(l, *conf, pluginRepo)) - cmd.AddCommand(jobValidateCommand(l, *conf, pluginRepo)) - cmd.AddCommand(jobRunCommand(l, *conf, pluginRepo)) + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + + cmd.AddCommand(jobCreateCommand(l, conf, pluginRepo)) + cmd.AddCommand(jobAddHookCommand(l, conf, pluginRepo)) + cmd.AddCommand(jobRenderTemplateCommand(l, conf, pluginRepo)) + cmd.AddCommand(jobValidateCommand(l, conf, pluginRepo)) + cmd.AddCommand(jobRunCommand(l, conf, pluginRepo)) cmd.AddCommand(jobRunListCommand(l, conf.Project.Name, conf.Host)) return cmd } diff --git a/cmd/job_create.go b/cmd/job_create.go index 01fe4fd16e..a2922e095f 100644 --- a/cmd/job_create.go +++ b/cmd/job_create.go @@ -33,7 +33,7 @@ var ( specFileNames = []string{local.ResourceSpecFileName, local.JobSpecFileName} ) -func jobCreateCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobCreateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new Job", diff --git a/cmd/job_hook.go b/cmd/job_hook.go index bb450ec85f..7d1afd19fc 100644 --- a/cmd/job_hook.go +++ b/cmd/job_hook.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobAddHookCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobAddHookCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "addhook", Aliases: []string{"add_hook", "add-hook", "addHook", "attach_hook", "attach-hook", "attachHook"}, diff --git a/cmd/job_render.go b/cmd/job_render.go index 0d3df10cb2..0634dcefb9 100644 --- a/cmd/job_render.go +++ b/cmd/job_render.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobRenderTemplateCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobRenderTemplateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { cmd := &cli.Command{ Use: "render", Short: "Apply template values in job specification to current 'render' directory", diff --git a/cmd/job_run.go b/cmd/job_run.go index e8fa6e31bb..923f631af4 100644 --- a/cmd/job_run.go +++ b/cmd/job_run.go @@ -22,7 +22,7 @@ const ( runJobTimeout = time.Minute * 1 ) -func jobRunCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobRunCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { var ( namespaceName string projectName = conf.Project.Name diff --git a/cmd/job_validate.go b/cmd/job_validate.go index ee0ea4525e..a2e19c216f 100644 --- a/cmd/job_validate.go +++ b/cmd/job_validate.go @@ -23,7 +23,7 @@ const ( validateTimeout = time.Minute * 5 ) -func jobValidateCommand(l log.Logger, conf config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobValidateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { var ( verbose bool namespaceName string diff --git a/cmd/logger.go b/cmd/logger.go index c88a5e4bdf..2dab93b3c3 100644 --- a/cmd/logger.go +++ b/cmd/logger.go @@ -9,15 +9,11 @@ import ( "github.com/sirupsen/logrus" ) -// conf -> client / server / ,, , ,, -// logger -> json / plain / xml / ... -// func -> 4 - type loggerType int const ( - jsonLogger loggerType = iota - plainLogger + jsonLoggerType loggerType = iota + plainLoggerType ) type plainFormatter int @@ -39,14 +35,14 @@ func initLogger(t loggerType, conf config.LogConfig) log.Logger { } switch t { - case jsonLogger: + case jsonLoggerType: return log.NewLogrus( - log.LogrusWithLevel(conf.Level), + log.LogrusWithLevel(conf.Level.String()), log.LogrusWithWriter(os.Stderr), ) - case plainLogger: + case plainLoggerType: return log.NewLogrus( - log.LogrusWithLevel(conf.Level), + log.LogrusWithLevel(conf.Level.String()), log.LogrusWithFormatter(new(plainFormatter)), ) } diff --git a/cmd/namespace.go b/cmd/namespace.go index 1d88c15d9b..a8448e789e 100644 --- a/cmd/namespace.go +++ b/cmd/namespace.go @@ -9,7 +9,7 @@ import ( "github.com/odpf/optimus/config" ) -func askToSelectNamespace(l log.Logger, conf config.ClientConfig) (*config.Namespace, error) { +func askToSelectNamespace(l log.Logger, conf *config.ClientConfig) (*config.Namespace, error) { options := make([]string, len(conf.Namespaces)) if len(conf.Namespaces) == 0 { return nil, errors.New("no namespace found in config file") diff --git a/cmd/replay.go b/cmd/replay.go index c545eba8ff..40f74fe9fe 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -52,7 +52,11 @@ func formatRunsPerJobInstance(instance *pb.ReplayExecutionTreeNode, taskReruns m } } -func replayCommand(l log.Logger) *cli.Command { +func replayCommand() *cli.Command { + var configFilePath string + var conf = &config.ClientConfig{} + var l log.Logger = initLogger(plainLoggerType, conf.Log) + cmd := &cli.Command{ Use: "replay", Short: "Re-running jobs in order to update data for older dates/partitions", @@ -61,16 +65,23 @@ func replayCommand(l log.Logger) *cli.Command { "group:core": "true", }, } + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + var err error + + conf, err = config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + l = initLogger(plainLoggerType, conf.Log) - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) return nil } - cmd.AddCommand(replayCreateCommand(l, *conf)) - cmd.AddCommand(replayStatusCommand(l, *conf)) - cmd.AddCommand(replayListCommand(l, *conf)) + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + + cmd.AddCommand(replayCreateCommand(l, conf)) + cmd.AddCommand(replayStatusCommand(l, conf)) + cmd.AddCommand(replayListCommand(l, conf)) return cmd } diff --git a/cmd/replay_create.go b/cmd/replay_create.go index aa35930dd6..8de5a611bc 100644 --- a/cmd/replay_create.go +++ b/cmd/replay_create.go @@ -19,7 +19,7 @@ import ( "github.com/odpf/optimus/core/set" ) -func replayCreateCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func replayCreateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( dryRun = false forceRun = false diff --git a/cmd/replay_list.go b/cmd/replay_list.go index 93669d5194..17b5494d54 100644 --- a/cmd/replay_list.go +++ b/cmd/replay_list.go @@ -15,7 +15,7 @@ import ( "github.com/odpf/optimus/models" ) -func replayListCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func replayListCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( projectName string reCmd = &cli.Command{ diff --git a/cmd/replay_status.go b/cmd/replay_status.go index 12a7f77b23..128c175ce6 100644 --- a/cmd/replay_status.go +++ b/cmd/replay_status.go @@ -15,7 +15,7 @@ import ( "github.com/odpf/optimus/models" ) -func replayStatusCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func replayStatusCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( projectName string ) diff --git a/cmd/resource.go b/cmd/resource.go index cdc40a9bca..20b39cf537 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -23,7 +23,11 @@ import ( var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-zA-Z0-9_\-\.]+$`, `invalid name (can only contain characters A-Z (in either case), 0-9, "-", "_" or "." and must start with an alphanumeric character)`) -func resourceCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Command { +func resourceCommand(datastoreRepo models.DatastoreRepo) *cli.Command { + var configFilePath string + var conf = &config.ClientConfig{} + var l log.Logger = initLogger(plainLoggerType, conf.Log) + cmd := &cli.Command{ Use: "resource", Short: "Interact with data resource", @@ -31,117 +35,124 @@ func resourceCommand(l log.Logger, datastoreRepo models.DatastoreRepo) *cli.Comm "group:core": "true", }, } + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + var err error - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) - return nil - } - - //init local specs - datastoreSpecFs := make(map[string]map[string]afero.Fs) - for _, namespace := range conf.Namespaces { - dtSpec := make(map[string]afero.Fs) - for _, dsConfig := range namespace.Datastore { - dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) + conf, err = config.LoadClientConfig(configFilePath) + if err != nil { + return err } - datastoreSpecFs[namespace.Name] = dtSpec + l = initLogger(plainLoggerType, conf.Log) + + return nil } - cmd.AddCommand(createResourceSubCommand(l, *conf, datastoreSpecFs, datastoreRepo)) + cmd.AddCommand(createResourceSubCommand(l, conf, datastoreRepo)) return cmd } -func createResourceSubCommand(l log.Logger, conf config.ClientConfig, datastoreSpecFs map[string]map[string]afero.Fs, datastoreRepo models.DatastoreRepo) *cli.Command { +func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new resource", Example: "optimus resource create", - RunE: func(cmd *cli.Command, args []string) error { - namespace, err := askToSelectNamespace(l, conf) - if err != nil { - return err - } - availableStorer := []string{} - for _, s := range datastoreRepo.GetAll() { - availableStorer = append(availableStorer, s.Name()) - } - var storerName string - if err := survey.AskOne(&survey.Select{ - Message: "Select supported datastores?", - Options: availableStorer, - }, &storerName); err != nil { - return err - } - repoFS, ok := datastoreSpecFs[namespace.Name][storerName] - if !ok { - return fmt.Errorf("unregistered datastore, please use configuration file to set datastore path") - } + } - // find requested datastore - availableTypes := []string{} - datastore, _ := datastoreRepo.GetByName(storerName) - for dsType := range datastore.Types() { - availableTypes = append(availableTypes, dsType.String()) + cmd.RunE = func(cmd *cli.Command, args []string) error { + //init local specs + datastoreSpecFs := make(map[string]map[string]afero.Fs) + for _, namespace := range conf.Namespaces { + dtSpec := make(map[string]afero.Fs) + for _, dsConfig := range namespace.Datastore { + dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) } - resourceSpecRepo := local.NewResourceSpecRepository(repoFS, datastore) - - // find resource type - var resourceType string - if err := survey.AskOne(&survey.Select{ - Message: "Select supported resource type?", - Options: availableTypes, - }, &resourceType); err != nil { - return err - } - typeController := datastore.Types()[models.ResourceType(resourceType)] + datastoreSpecFs[namespace.Name] = dtSpec + } - // find directory to store spec - rwd, err := getWorkingDirectory(repoFS, "") - if err != nil { - return err - } - newDirName, err := getDirectoryName(rwd) - if err != nil { - return err - } + namespace, err := askToSelectNamespace(l, conf) + if err != nil { + return err + } + availableStorer := []string{} + for _, s := range datastoreRepo.GetAll() { + availableStorer = append(availableStorer, s.Name()) + } + var storerName string + if err := survey.AskOne(&survey.Select{ + Message: "Select supported datastores?", + Options: availableStorer, + }, &storerName); err != nil { + return err + } + repoFS, ok := datastoreSpecFs[namespace.Name][storerName] + if !ok { + return fmt.Errorf("unregistered datastore, please use configuration file to set datastore path") + } + + // find requested datastore + availableTypes := []string{} + datastore, _ := datastoreRepo.GetByName(storerName) + for dsType := range datastore.Types() { + availableTypes = append(availableTypes, dsType.String()) + } + resourceSpecRepo := local.NewResourceSpecRepository(repoFS, datastore) + + // find resource type + var resourceType string + if err := survey.AskOne(&survey.Select{ + Message: "Select supported resource type?", + Options: availableTypes, + }, &resourceType); err != nil { + return err + } + typeController := datastore.Types()[models.ResourceType(resourceType)] - resourceDirectory := filepath.Join(rwd, newDirName) - resourceNameDefault := strings.ReplaceAll(strings.ReplaceAll(resourceDirectory, "/", "."), "\\", ".") - - qs := []*survey.Question{ - { - Name: "name", - Prompt: &survey.Input{ - Message: "What is the resource name?(should conform to selected resource type)", - Default: resourceNameDefault, - }, - Validate: survey.ComposeValidators(validateNoSlash, survey.MinLength(3), - survey.MaxLength(1024), IsValidDatastoreSpec(typeController.Validator()), - IsResourceNameUnique(resourceSpecRepo)), + // find directory to store spec + rwd, err := getWorkingDirectory(repoFS, "") + if err != nil { + return err + } + newDirName, err := getDirectoryName(rwd) + if err != nil { + return err + } + + resourceDirectory := filepath.Join(rwd, newDirName) + resourceNameDefault := strings.ReplaceAll(strings.ReplaceAll(resourceDirectory, "/", "."), "\\", ".") + + var qs = []*survey.Question{ + { + Name: "name", + Prompt: &survey.Input{ + Message: "What is the resource name?(should conform to selected resource type)", + Default: resourceNameDefault, }, - } - inputs := map[string]interface{}{} - if err := survey.Ask(qs, &inputs); err != nil { - return err - } - resourceName := inputs["name"].(string) - - if err := resourceSpecRepo.SaveAt(models.ResourceSpec{ - Version: 1, - Name: resourceName, - Type: models.ResourceType(resourceType), - Datastore: datastore, - Assets: typeController.DefaultAssets(), - }, resourceDirectory); err != nil { - return err - } + Validate: survey.ComposeValidators(validateNoSlash, survey.MinLength(3), + survey.MaxLength(1024), IsValidDatastoreSpec(typeController.Validator()), + IsResourceNameUnique(resourceSpecRepo)), + }, + } + inputs := map[string]interface{}{} + if err := survey.Ask(qs, &inputs); err != nil { + return err + } + resourceName := inputs["name"].(string) + + if err := resourceSpecRepo.SaveAt(models.ResourceSpec{ + Version: 1, + Name: resourceName, + Type: models.ResourceType(resourceType), + Datastore: datastore, + Assets: typeController.DefaultAssets(), + }, resourceDirectory); err != nil { + return err + } - l.Info(coloredSuccess("Resource created successfully %s", resourceName)) - return nil - }, + l.Info(coloredSuccess("Resource created successfully %s", resourceName)) + return nil } + return cmd } diff --git a/cmd/secret.go b/cmd/secret.go index 8cfa48256b..6ba2adf012 100644 --- a/cmd/secret.go +++ b/cmd/secret.go @@ -26,26 +26,35 @@ const ( secretTimeout = time.Minute * 2 ) -func secretCommand(l log.Logger) *cli.Command { +func secretCommand() *cli.Command { + var configFilePath string + var conf = &config.ClientConfig{} + var l log.Logger = initLogger(plainLoggerType, conf.Log) + cmd := &cli.Command{ Use: "secret", Short: "Manage secrets to be used in jobs", } + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + var err error + + conf, err = config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + l = initLogger(plainLoggerType, conf.Log) - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - l.Error(err.Error()) return nil } - cmd.AddCommand(secretSetSubCommand(l, *conf)) - cmd.AddCommand(secretListSubCommand(l, *conf)) - cmd.AddCommand(secretDeleteSubCommand(l, *conf)) + cmd.AddCommand(secretSetSubCommand(l, conf)) + cmd.AddCommand(secretListSubCommand(l, conf)) + cmd.AddCommand(secretDeleteSubCommand(l, conf)) return cmd } -func secretSetSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func secretSetSubCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( projectName string namespaceName string @@ -130,7 +139,7 @@ Use base64 flag if the value has been encoded. return secretCmd } -func secretListSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func secretListSubCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var projectName string secretListCmd := &cli.Command{ @@ -150,7 +159,7 @@ func secretListSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { return secretListCmd } -func secretDeleteSubCommand(l log.Logger, conf config.ClientConfig) *cli.Command { +func secretDeleteSubCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var projectName, namespaceName string cmd := &cli.Command{ diff --git a/cmd/serve.go b/cmd/serve.go index 582268b915..6813d14ee1 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -31,8 +31,11 @@ func serveCommand() *cli.Command { } // initiate jsonLogger - l := initLogger(jsonLogger, conf.Log) + l := initLogger(jsonLoggerType, conf.Log) pluginLogLevel := hclog.Info + if conf.Log.Level == config.LogLevelDebug { + pluginLogLevel = hclog.Debug + } // discover and load plugins. TODO: refactor this if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ diff --git a/cmd/version.go b/cmd/version.go index f145fc118f..fcc0aa1016 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -5,14 +5,12 @@ import ( "fmt" "time" - "github.com/odpf/salt/log" - "github.com/odpf/salt/version" - cli "github.com/spf13/cobra" - "google.golang.org/grpc" - pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" + "github.com/odpf/salt/version" + cli "github.com/spf13/cobra" + "google.golang.org/grpc" ) const ( @@ -20,8 +18,9 @@ const ( githubRepo = "odpf/optimus" ) -func versionCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Command { +func versionCommand(pluginRepo models.PluginRepository) *cli.Command { var isWithServer bool + var configFilePath string c := &cli.Command{ Use: "version", @@ -29,20 +28,28 @@ func versionCommand(l log.Logger, pluginRepo models.PluginRepository) *cli.Comma Example: "optimus version [--with-server]", } - c.Flags().BoolVar(&isWithServer, "with-server", false, "Check for server version") + c.Flags().BoolVar(&isWithServer, "with-server", isWithServer, "Check for server version") + c.Flags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") c.RunE = func(c *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + conf, err := config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + + if err := config.Validate(conf); err != nil { + return err + } + + // initiate logger + l := initLogger(plainLoggerType, conf.Log) + // Print client version l.Info(fmt.Sprintf("Client: %s-%s", coloredNotice(config.BuildVersion), coloredNotice(config.BuildCommit))) // Print server version if isWithServer { - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig() - if err != nil { - return err - } - srvVer, err := getVersionRequest(config.BuildVersion, conf.Host) if err != nil { return err From de07b3d34fe5a13ea1a208da0f5a5fe79e72d817 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 16:02:32 +0700 Subject: [PATCH 19/54] fix: invalid home filepath on load server config test --- config/loader_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/loader_test.go b/config/loader_test.go index 3b57f074b2..0931c24781 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -88,7 +88,7 @@ func (s *ConfigTestSuite) SetupTest() { config.FS = s.a.Fs - s.initExpectedProjectConfig() + s.initExpectedClientConfig() s.initExpectedServerConfig() } @@ -156,7 +156,7 @@ func (s *ConfigTestSuite) TestLoadClientConfig() { func (s *ConfigTestSuite) TestLoadServerConfig() { execFilePath := path.Join(s.execPath, config.DefaultFilename+"."+config.DefaultFileExtension) - homeFilePath := path.Join(s.homePath, config.DefaultFilename+"."+config.DefaultFilename) + homeFilePath := path.Join(s.homePath, config.DefaultFilename+"."+config.DefaultFileExtension) s.a.WriteFile(execFilePath, []byte(serverConfig), fs.ModeTemporary) s.a.WriteFile(homeFilePath, []byte(`version: 3`), fs.ModeTemporary) s.initServerConfigEnv() @@ -178,7 +178,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { s.Assert().NoError(err) s.Assert().NotNil(conf) - s.Assert().Equal(s.expectedServerConfig, conf) // should load from exec dir + s.Assert().EqualValues(s.expectedServerConfig, conf) // should load from exec dir }) s.Run("WhenEnvNotExistAndExecDirNotExist", func() { @@ -226,7 +226,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { }) } -func (s *ConfigTestSuite) initExpectedProjectConfig() { +func (s *ConfigTestSuite) initExpectedClientConfig() { s.expectedClientConfig = &config.ClientConfig{} s.expectedClientConfig.Version = config.Version(1) s.expectedClientConfig.Log = config.LogConfig{Level: "info"} From d3088eeec48e0ba68291ee67a27901b118f3d344 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 16:02:53 +0700 Subject: [PATCH 20/54] fix: nested validation --- config/config_common.go | 16 +++--- config/validation.go | 32 +++++++++--- config/validation_test.go | 107 +++++++++++++------------------------- 3 files changed, 71 insertions(+), 84 deletions(-) diff --git a/config/config_common.go b/config/config_common.go index 216ba55057..ed160ab2db 100644 --- a/config/config_common.go +++ b/config/config_common.go @@ -14,17 +14,21 @@ type LogLevel string const ( LogLevelDebug LogLevel = "DEBUG" - LogLevelInfo = "INFO" - LogLevelWarning = "INFO" - LogLevelError = "ERROR" - LogLevelFatal = "FATAL" + LogLevelInfo LogLevel = "INFO" + LogLevelWarning LogLevel = "WARNING" + LogLevelError LogLevel = "ERROR" + LogLevelFatal LogLevel = "FATAL" ) type LogConfig struct { - Level string `mapstructure:"level" default:"info"` // log level - debug, info, warning, error, fatal - Format string `mapstructure:"format"` // format strategy - plain, json + Level LogLevel `mapstructure:"level" default:"INFO"` // log level - debug, info, warning, error, fatal + Format string `mapstructure:"format"` // format strategy - plain, json } func (v Version) String() string { return strconv.Itoa(int(v)) } + +func (l LogLevel) String() string { + return string(l) +} diff --git a/config/validation.go b/config/validation.go index 99d277eb10..b5182ec779 100644 --- a/config/validation.go +++ b/config/validation.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "reflect" "strings" validation "github.com/go-ozzo/ozzo-validation/v4" @@ -24,13 +25,15 @@ func validateClientConfig(conf ClientConfig) error { return validation.ValidateStruct(&conf, validation.Field(&conf.Version, validation.Required), validation.Field(&conf.Host, validation.Required), - validation.Field(&conf.Log.Level, validation.In( - LogLevelDebug, - LogLevelInfo, - LogLevelWarning, - LogLevelError, - LogLevelFatal, - )), + nestedFields(&conf.Log, + validation.Field(&conf.Log.Level, validation.In( + LogLevelDebug, + LogLevelInfo, + LogLevelWarning, + LogLevelError, + LogLevelFatal, + )), + ), validation.Field(&conf.Namespaces, validation.By(validateNamespaces)), // ... etc ) @@ -44,7 +47,7 @@ func validateServerConfig(conf ServerConfig) error { func validateNamespaces(value interface{}) error { namespaces, ok := value.([]*Namespace) if !ok { - return errors.New("error") + return errors.New("can't convert value to namespaces") } m := map[string]int{} @@ -68,3 +71,16 @@ func validateNamespaces(value interface{}) error { return nil } + +// ozzo-validation helper for nested validation struct +// https://github.com/go-ozzo/ozzo-validation/issues/136 +func nestedFields(target interface{}, fieldRules ...*validation.FieldRules) *validation.FieldRules { + return validation.Field(target, validation.By(func(value interface{}) error { + valueV := reflect.Indirect(reflect.ValueOf(value)) + if valueV.CanAddr() { + addr := valueV.Addr().Interface() + return validation.ValidateStruct(addr, fieldRules...) + } + return validation.ValidateStruct(target, fieldRules...) + })) +} diff --git a/config/validation_test.go b/config/validation_test.go index ea9e69e6e0..17e5697cde 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -1,97 +1,63 @@ -package config +package config_test import ( "testing" + "github.com/odpf/optimus/config" "github.com/stretchr/testify/suite" ) type ValidationTestSuite struct { suite.Suite - defaultProjectConfig ClientConfig + defaultClientConfig config.ClientConfig } func (s *ValidationTestSuite) SetupTest() { - s.initDefaultProjectConfig() + s.initDefaultClientConfig() } func TestValidation(t *testing.T) { suite.Run(t, new(ValidationTestSuite)) } -func (s *ValidationTestSuite) TestInternal_ValidateNamespaces_Success() { - namespaces := []*Namespace{ - { - Name: "namespace-1", - Job: Job{Path: "path-1"}, - }, { - Name: "namespace-2", - Job: Job{Path: "path-2"}, - }, - nil, - } +func (s *ValidationTestSuite) TestValidate() { + s.Run("WhenValidateClientConfig", func() { + s.Run("WhenConfigIsValid", func() { + err := config.Validate(s.defaultClientConfig) + s.Assert().NoError(err) + }) - err := validateNamespaces(namespaces) - s.Assert().NoError(err) -} + s.Run("WhenNamespacesIsDuplicated", func() { + clientConfig := s.defaultClientConfig + namespaces := clientConfig.Namespaces + namespaces = append(namespaces, &config.Namespace{Name: "ns-dup"}) + namespaces = append(namespaces, &config.Namespace{Name: "ns-dup"}) + clientConfig.Namespaces = namespaces -func (s *ValidationTestSuite) TestInternal_ValidateNamespaces_Fail() { - s.Run("WhenTypeAssertionIsFailed", func() { - invalidStruct := "this-is-string" - err := validateNamespaces(invalidStruct) - s.Assert().Error(err) - }) - - s.Run("WhenDuplicationIsDetected", func() { - namespaces := []*Namespace{ - { - Name: "other-ns", - Job: Job{Path: "path-other"}, - }, { - Name: "dup-ns", - Job: Job{Path: "path-1"}, - }, { - Name: "dup-ns", - Job: Job{Path: "path-2"}, - }, - } + err := config.Validate(clientConfig) - err := validateNamespaces(namespaces) - s.Assert().Error(err) + s.Assert().Error(err) + }) }) -} -func (s *ValidationTestSuite) TestValidate_ProjectConfig() { - s.Run("WhenStructIsValid", func() { - err := Validate(s.defaultProjectConfig) - s.Assert().NoError(err) + s.Run("WhenValidateServerConfig", func() { + // TODO: implement this + s.T().Skip() }) - s.Run("WhenStructIsInvalid", func() { - conf := s.defaultProjectConfig - conf.Host = "" - - err := Validate(conf) + s.Run("WhenValidateTypeIsInvalid", func() { + err := config.Validate("invalid-type") s.Assert().Error(err) }) } -func (s *ValidationTestSuite) TestValidate_ServerConfig() { - // TODO: implement this -} - -func (s *ValidationTestSuite) TestValidate_Fail() { - err := Validate("invalid-type") - s.Assert().Error(err) -} - -func (s *ValidationTestSuite) initDefaultProjectConfig() { - s.defaultProjectConfig = ClientConfig{} - s.defaultProjectConfig.Version = Version(1) - s.defaultProjectConfig.Log = LogConfig{Level: "info"} +func (s *ValidationTestSuite) initDefaultClientConfig() { + s.defaultClientConfig = config.ClientConfig{} + s.defaultClientConfig.Version = config.Version(1) + s.defaultClientConfig.Log = config.LogConfig{Level: config.LogLevelInfo} - s.defaultProjectConfig.Host = "localhost:9100" - s.defaultProjectConfig.Project = Project{ + s.defaultClientConfig.Host = "localhost:9100" + s.defaultClientConfig.Project = config.Project{ Name: "sample_project", Config: map[string]string{ "environment": "integration", @@ -99,18 +65,19 @@ func (s *ValidationTestSuite) initDefaultProjectConfig() { "storage_path": "file://absolute_path_to_a_directory", }, } - namespaces := []*Namespace{} - namespaces = append(namespaces, &Namespace{ + namespaces := []*config.Namespace{} + namespaces = append(namespaces, &config.Namespace{ Name: "namespace-a", - Job: Job{ + Job: config.Job{ Path: "./jobs-a", }, }) - namespaces = append(namespaces, &Namespace{ + namespaces = append(namespaces, &config.Namespace{ Name: "namespace-b", - Job: Job{ + Job: config.Job{ Path: "./jobs-b", }, }) - s.defaultProjectConfig.Namespaces = namespaces + namespaces = append(namespaces, nil) + s.defaultClientConfig.Namespaces = namespaces } From eda5818c62cd01fa9ef513a3554d83365faa01a3 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 17:15:14 +0700 Subject: [PATCH 21/54] fix: linter --- cmd/admin.go | 4 ++-- cmd/backup.go | 8 +++----- cmd/config.go | 5 +++-- cmd/deploy.go | 2 +- cmd/extension.go | 3 +-- cmd/job.go | 6 ++---- cmd/replay.go | 8 +++----- cmd/replay_status.go | 4 +--- cmd/resource.go | 8 ++++---- cmd/secret.go | 4 ++-- cmd/serve.go | 2 +- config/loader.go | 1 + config/loader_test.go | 6 +++--- go.mod | 1 - 14 files changed, 27 insertions(+), 35 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index d415cd8a6f..eff220e96e 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -10,8 +10,8 @@ import ( // adminCommand registers internal administration commands func adminCommand() *cli.Command { var configFilePath string - var conf = &config.ClientConfig{} - var l log.Logger = initLogger(plainLoggerType, conf.Log) + conf := &config.ClientConfig{} + l := initLogger(plainLoggerType, conf.Log) cmd := &cli.Command{ Use: "admin", diff --git a/cmd/backup.go b/cmd/backup.go index 2ae103f1ee..6bad0853b4 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -4,11 +4,9 @@ import ( "time" "github.com/MakeNowJust/heredoc" - "github.com/odpf/salt/log" - cli "github.com/spf13/cobra" - "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" + cli "github.com/spf13/cobra" ) const ( @@ -17,8 +15,8 @@ const ( func backupCommand(datastoreRepo models.DatastoreRepo) *cli.Command { var configFilePath string - var conf = &config.ClientConfig{} - var l log.Logger = initLogger(plainLoggerType, conf.Log) + conf := &config.ClientConfig{} + l := initLogger(plainLoggerType, conf.Log) cmd := &cli.Command{ Use: "backup", diff --git a/cmd/config.go b/cmd/config.go index 7818d9a389..7817b167b1 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -11,7 +11,8 @@ import ( ) const ( - defaultHost = "localhost" + defaultHost = "localhost" + defaultFilePermissionMode = 0o655 ) func configCommand() *cli.Command { @@ -91,7 +92,7 @@ func configInitCommand() *cli.Command { if err != nil { return err } - if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension), confMarshaled, 0655); err != nil { + if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension), confMarshaled, defaultFilePermissionMode); err != nil { return err } diff --git a/cmd/deploy.go b/cmd/deploy.go index 81d42ed2ca..55ef326410 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -63,7 +63,7 @@ func deployCommand(pluginRepo models.PluginRepository, dsRepo models.DatastoreRe // init logger l := initLogger(plainLoggerType, conf.Log) - //init local specs + // init local specs datastoreSpecFs := make(map[string]map[string]afero.Fs) for _, namespace := range conf.Namespaces { dtSpec := make(map[string]afero.Fs) diff --git a/cmd/extension.go b/cmd/extension.go index 804dfb87e7..fd73844a5a 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -10,7 +10,6 @@ import ( "github.com/google/go-github/github" "github.com/odpf/optimus/config" "github.com/odpf/optimus/extension" - "github.com/odpf/salt/log" cli "github.com/spf13/cobra" ) @@ -55,7 +54,7 @@ func extensionCommand(ctx context.Context, extension *extension.Extension) *cli. func extensionInstallCommand(ctx context.Context, installer extension.Installer) *cli.Command { var ( alias string - l log.Logger = initLogger(plainLoggerType, config.LogConfig{}) + l = initLogger(plainLoggerType, config.LogConfig{}) ) installCmd := &cli.Command{ diff --git a/cmd/job.go b/cmd/job.go index 814b28f99e..2e11263211 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -3,17 +3,15 @@ package cmd import ( "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" - "github.com/odpf/salt/log" cli "github.com/spf13/cobra" ) func jobCommand(pluginRepo models.PluginRepository) *cli.Command { var configFilePath string - var conf = &config.ClientConfig{} - var l log.Logger = initLogger(plainLoggerType, conf.Log) + conf := &config.ClientConfig{} + l := initLogger(plainLoggerType, conf.Log) cmd := &cli.Command{ - Use: "job", Short: "Interact with schedulable Job", Annotations: map[string]string{ diff --git a/cmd/replay.go b/cmd/replay.go index 40f74fe9fe..961aa39639 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -4,12 +4,10 @@ import ( "strings" "time" - "github.com/odpf/salt/log" - cli "github.com/spf13/cobra" - pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" "github.com/odpf/optimus/config" "github.com/odpf/optimus/core/set" + cli "github.com/spf13/cobra" ) const ( @@ -54,8 +52,8 @@ func formatRunsPerJobInstance(instance *pb.ReplayExecutionTreeNode, taskReruns m func replayCommand() *cli.Command { var configFilePath string - var conf = &config.ClientConfig{} - var l log.Logger = initLogger(plainLoggerType, conf.Log) + conf := &config.ClientConfig{} + l := initLogger(plainLoggerType, conf.Log) cmd := &cli.Command{ Use: "replay", diff --git a/cmd/replay_status.go b/cmd/replay_status.go index 128c175ce6..b5c466730d 100644 --- a/cmd/replay_status.go +++ b/cmd/replay_status.go @@ -16,9 +16,7 @@ import ( ) func replayStatusCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { - var ( - projectName string - ) + var projectName string reCmd := &cli.Command{ Use: "status", diff --git a/cmd/resource.go b/cmd/resource.go index 20b39cf537..bc3018ea2e 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -25,8 +25,8 @@ var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-z func resourceCommand(datastoreRepo models.DatastoreRepo) *cli.Command { var configFilePath string - var conf = &config.ClientConfig{} - var l log.Logger = initLogger(plainLoggerType, conf.Log) + conf := &config.ClientConfig{} + l := initLogger(plainLoggerType, conf.Log) cmd := &cli.Command{ Use: "resource", @@ -60,7 +60,7 @@ func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastore } cmd.RunE = func(cmd *cli.Command, args []string) error { - //init local specs + // init local specs datastoreSpecFs := make(map[string]map[string]afero.Fs) for _, namespace := range conf.Namespaces { dtSpec := make(map[string]afero.Fs) @@ -121,7 +121,7 @@ func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastore resourceDirectory := filepath.Join(rwd, newDirName) resourceNameDefault := strings.ReplaceAll(strings.ReplaceAll(resourceDirectory, "/", "."), "\\", ".") - var qs = []*survey.Question{ + qs := []*survey.Question{ { Name: "name", Prompt: &survey.Input{ diff --git a/cmd/secret.go b/cmd/secret.go index 6ba2adf012..4dbf4d9cf4 100644 --- a/cmd/secret.go +++ b/cmd/secret.go @@ -28,8 +28,8 @@ const ( func secretCommand() *cli.Command { var configFilePath string - var conf = &config.ClientConfig{} - var l log.Logger = initLogger(plainLoggerType, conf.Log) + conf := &config.ClientConfig{} + l := initLogger(plainLoggerType, conf.Log) cmd := &cli.Command{ Use: "secret", diff --git a/cmd/serve.go b/cmd/serve.go index 6813d14ee1..5daed7d8cc 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -58,7 +58,7 @@ func serveCommand() *cli.Command { } defer teleShutdown() l.Info(coloredSuccess("Starting Optimus"), "version", config.BuildVersion) - optimusServer, err := server.New(l, conf) + optimusServer, err := server.New(l, *conf) defer optimusServer.Shutdown() if err != nil { return fmt.Errorf("unable to create server: %w", err) diff --git a/config/loader.go b/config/loader.go index a7a84f21c6..b31d4c62c1 100644 --- a/config/loader.go +++ b/config/loader.go @@ -27,6 +27,7 @@ var ( homePath string ) +//nolint:gochecknoinits func init() { p, err := os.Getwd() if err != nil { diff --git a/config/loader_test.go b/config/loader_test.go index 0931c24781..65645e20f8 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -32,6 +32,7 @@ namespaces: job: path: ./jobs-b ` + const serverConfig = ` version: 1 log: @@ -151,7 +152,6 @@ func (s *ConfigTestSuite) TestLoadClientConfig() { s.Assert().Nil(conf) }) }) - } func (s *ConfigTestSuite) TestLoadServerConfig() { @@ -219,7 +219,8 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { }) s.Run("WhenFilePathIsNotValid", func() { - conf, err := config.LoadServerConfig("/path/not/exist") + s.a.MkdirAll("/path/dir/", os.ModeTemporary) + conf, err := config.LoadServerConfig("/path/dir/") s.Assert().Error(err) s.Assert().Nil(conf) }) @@ -319,5 +320,4 @@ func (s *ConfigTestSuite) unsetServerConfigEnv() { os.Unsetenv("OPTIMUS_SCHEDULER_SKIP_INIT") os.Unsetenv("OPTIMUS_TELEMETRY_PROFILE_ADDR") os.Unsetenv("OPTIMUS_TELEMETRY_JAEGER_ADDR") - } diff --git a/go.mod b/go.mod index 5cd0c48e61..0931f5502e 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,6 @@ require ( github.com/Masterminds/semver/v3 v3.1.1 // indirect github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect - github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/boltdb/bolt v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect From 6f03207f259d45c919df0269df21e26122d971c7 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 17:39:05 +0700 Subject: [PATCH 22/54] fix: handled validation receive pointer --- config/loader.go | 2 ++ config/validation.go | 12 ++++++------ config/validation_test.go | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/config/loader.go b/config/loader.go index b31d4c62c1..0f3251b534 100644 --- a/config/loader.go +++ b/config/loader.go @@ -148,6 +148,8 @@ func LoadClientConfig(filePath string) (*ClientConfig, error) { return nil, err } + cfg.Log.Level = LogLevel(strings.ToUpper(string(cfg.Log.Level))) + return cfg, nil } diff --git a/config/validation.go b/config/validation.go index b5182ec779..1da73c02e9 100644 --- a/config/validation.go +++ b/config/validation.go @@ -12,17 +12,17 @@ import ( // Validate validate the config as an input. If not valid, it returns error func Validate(conf Config) error { switch c := conf.(type) { - case ClientConfig: + case *ClientConfig: return validateClientConfig(c) - case ServerConfig: + case *ServerConfig: return validateServerConfig(c) } - return errors.New("error") + return errors.New("config type is not valid, use ClientConfig or ServerConfig instead") } -func validateClientConfig(conf ClientConfig) error { +func validateClientConfig(conf *ClientConfig) error { // implement this - return validation.ValidateStruct(&conf, + return validation.ValidateStruct(conf, validation.Field(&conf.Version, validation.Required), validation.Field(&conf.Host, validation.Required), nestedFields(&conf.Log, @@ -39,7 +39,7 @@ func validateClientConfig(conf ClientConfig) error { ) } -func validateServerConfig(conf ServerConfig) error { +func validateServerConfig(conf *ServerConfig) error { // implement this return nil } diff --git a/config/validation_test.go b/config/validation_test.go index 17e5697cde..983202434a 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -9,7 +9,7 @@ import ( type ValidationTestSuite struct { suite.Suite - defaultClientConfig config.ClientConfig + defaultClientConfig *config.ClientConfig } func (s *ValidationTestSuite) SetupTest() { @@ -52,7 +52,7 @@ func (s *ValidationTestSuite) TestValidate() { } func (s *ValidationTestSuite) initDefaultClientConfig() { - s.defaultClientConfig = config.ClientConfig{} + s.defaultClientConfig = &config.ClientConfig{} s.defaultClientConfig.Version = config.Version(1) s.defaultClientConfig.Log = config.LogConfig{Level: config.LogLevelInfo} From 6d73cf7c87f2e5a643b095ad35f33f57e46c9820 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 17:46:18 +0700 Subject: [PATCH 23/54] fix: smoke-test script to use sample config --- optimus.sample.yaml | 2 +- scripts/smoke-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/optimus.sample.yaml b/optimus.sample.yaml index 25d12f852a..6de14f6803 100644 --- a/optimus.sample.yaml +++ b/optimus.sample.yaml @@ -10,7 +10,7 @@ log: level: info # used to connect optimus service -#host: localhost:9100 +host: localhost:9100 # for configuring optimus project #project: diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index cac5e2ebb9..8011d1e4ca 100644 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -1,6 +1,6 @@ #!/bin/sh -./optimus version &> /dev/null +./optimus version -c ./optimus.sample.yaml &> /dev/null STATUS=$? if [ ! $STATUS -eq 0 ]; then echo "[smoke test] FAIL: optimus exited with code ${STATUS}"; From 273692de36a321fe5585944efb26e9bb2c13480c Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 24 Mar 2022 17:55:35 +0700 Subject: [PATCH 24/54] fix: test on load log level --- config/loader.go | 2 ++ config/loader_test.go | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/loader.go b/config/loader.go index 0f3251b534..1b1d5260da 100644 --- a/config/loader.go +++ b/config/loader.go @@ -191,6 +191,8 @@ func LoadServerConfig(filePath string) (*ServerConfig, error) { return nil, err } + cfg.Log.Level = LogLevel(strings.ToUpper(string(cfg.Log.Level))) + return cfg, nil } diff --git a/config/loader_test.go b/config/loader_test.go index 65645e20f8..407cd36b08 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -230,7 +230,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { func (s *ConfigTestSuite) initExpectedClientConfig() { s.expectedClientConfig = &config.ClientConfig{} s.expectedClientConfig.Version = config.Version(1) - s.expectedClientConfig.Log = config.LogConfig{Level: "info"} + s.expectedClientConfig.Log = config.LogConfig{Level: config.LogLevelInfo} s.expectedClientConfig.Host = "localhost:9100" s.expectedClientConfig.Project = config.Project{ @@ -260,7 +260,7 @@ func (s *ConfigTestSuite) initExpectedClientConfig() { func (s *ConfigTestSuite) initExpectedServerConfig() { s.expectedServerConfig = &config.ServerConfig{} s.expectedServerConfig.Version = config.Version(1) - s.expectedServerConfig.Log = config.LogConfig{Level: "info"} + s.expectedServerConfig.Log = config.LogConfig{Level: config.LogLevelInfo} s.expectedServerConfig.Serve = config.Serve{} s.expectedServerConfig.Serve.Port = 9100 From a90e158eec0e27e1322bef58f3a08147627b1709 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 25 Mar 2022 07:43:08 +0700 Subject: [PATCH 25/54] test: cherry-pick legacy loader test --- config/loader.go | 2 +- config/loader_test.go | 143 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 1 deletion(-) diff --git a/config/loader.go b/config/loader.go index 1b1d5260da..31286cf9b8 100644 --- a/config/loader.go +++ b/config/loader.go @@ -104,7 +104,7 @@ func loadConfig(cfg interface{}, fs afero.Fs, dirPath string) error { opts := []config.LoaderOption{ config.WithViper(v), - config.WithName(DefaultFilename), + config.WithName(".optimus"), config.WithType(DefaultFileExtension), config.WithEnvPrefix("OPTIMUS"), config.WithEnvKeyReplacer(".", "_"), diff --git a/config/loader_test.go b/config/loader_test.go index 407cd36b08..a651bd2cc9 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -10,6 +10,7 @@ import ( "github.com/odpf/optimus/config" "github.com/spf13/afero" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -57,6 +58,148 @@ telemetry: jaeger_addr: "http://localhost:14268/api/traces" ` +// (LEGACY) +const ( + configFileName = ".optimus.yaml" + optimusConfigDirName = "./optimus" +) + +// (LEGACY) +const optimusConfigContent = ` +version: 1 +log: + level: info +host: "localhost:9100" +project: + name: sample_project + config: + environment: integration + scheduler_host: http://example.io/ + storage_path: file://absolute_path_to_a_directory +serve: + port: 9100 + host: localhost + ingress_host: optimus.example.io:80 + app_key: Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc + replay_num_workers: 1 + replay_worker_timeout: "100s" + replay_run_timeout: "10s" + db: + dsn: postgres://user:password@localhost:5432/database?sslmode=disable + max_idle_connection: 5 + max_open_connection: 10 +scheduler: + name: airflow2 + skip_init: true +telemetry: + profile_addr: ":9110" + jaeger_addr: "http://localhost:14268/api/traces" +namespaces: +- name: namespace-a + job: + path: ./jobs-a +- name: namespace-b + job: + path: ./jobs-b +` + +// (LEGACY) +func setup(content string) { + teardown() + if err := os.Mkdir(optimusConfigDirName, 0o750); err != nil { + panic(err) + } + confPath := path.Join(optimusConfigDirName, configFileName) + if err := os.WriteFile(confPath, []byte(content), 0o660); err != nil { + panic(err) + } +} + +func teardown() { + if err := os.RemoveAll(optimusConfigDirName); err != nil { + panic(err) + } +} + +// (LEGACY) +func TestLoadOptimusConfig(t *testing.T) { + t.Run("should return config and nil if no error is found", func(t *testing.T) { + setup(optimusConfigContent + `- name: namespace-b + job: + path: ./jobs-b`) + defer teardown() + + expectedErrMsg := "namespaces [namespace-b] are duplicate" + + actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) + + assert.Nil(t, actualConf) + assert.EqualError(t, actualErr, expectedErrMsg) + }) + + t.Run("should return config and nil if no error is found", func(t *testing.T) { + setup(optimusConfigContent) + defer teardown() + + expectedConf := &config.Optimus{ + Version: 1, + Log: config.LogConfig{ + Level: "info", + }, + Host: "localhost:9100", + Project: config.Project{ + Name: "sample_project", + Config: map[string]string{ + "environment": "integration", + "scheduler_host": "http://example.io/", + "storage_path": "file://absolute_path_to_a_directory", + }, + }, + Server: config.Serve{ + Port: 9100, + Host: "localhost", + IngressHost: "optimus.example.io:80", + AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", + ReplayNumWorkers: 1, + ReplayWorkerTimeout: 100 * time.Second, + ReplayRunTimeout: 10 * time.Second, + DB: config.DBConfig{ + DSN: "postgres://user:password@localhost:5432/database?sslmode=disable", + MaxIdleConnection: 5, + MaxOpenConnection: 10, + }, + }, + Scheduler: config.SchedulerConfig{ + Name: "airflow2", + SkipInit: true, + }, + Telemetry: config.TelemetryConfig{ + ProfileAddr: ":9110", + JaegerAddr: "http://localhost:14268/api/traces", + }, + Namespaces: []*config.Namespace{ + { + Name: "namespace-a", + Job: config.Job{ + Path: "./jobs-a", + }, + }, + { + Name: "namespace-b", + Job: config.Job{ + Path: "./jobs-b", + }, + }, + }, + } + + actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) + + assert.EqualValues(t, expectedConf, actualConf) + assert.NoError(t, actualErr) + }) +} + type ConfigTestSuite struct { suite.Suite a afero.Afero From f640ae961611b700cd0859a999f2edd3ddd6751e Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 25 Mar 2022 07:54:21 +0700 Subject: [PATCH 26/54] fix: legacy test race condition --- config/loader_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/config/loader_test.go b/config/loader_test.go index a651bd2cc9..ed4b00d1bd 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -113,6 +113,7 @@ func setup(content string) { if err := os.WriteFile(confPath, []byte(content), 0o660); err != nil { panic(err) } + unsetServerConfigEnv() } func teardown() { @@ -311,6 +312,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { s.Assert().NoError(err) s.Assert().NotNil(conf) s.Assert().Equal("4", conf.Version.String()) // should load from env var + s.Assert().Equal("INFO", conf.Log.Level.String()) }) s.Run("WhenEnvNotExist", func() { @@ -335,6 +337,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { s.Assert().NoError(err) s.Assert().NotNil(conf) s.Assert().Equal("3", conf.Version.String()) // should load from home dir + s.Assert().Equal("INFO", conf.Log.Level.String()) }) s.Run("WhenConfigNotFound", func() { @@ -359,6 +362,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { s.Assert().NoError(err) s.Assert().NotNil(conf) s.Assert().Equal("2", conf.Version.String()) + s.Assert().Equal("INFO", conf.Log.Level.String()) }) s.Run("WhenFilePathIsNotValid", func() { @@ -447,6 +451,10 @@ func (s *ConfigTestSuite) initServerConfigEnv() { } func (s *ConfigTestSuite) unsetServerConfigEnv() { + unsetServerConfigEnv() +} + +func unsetServerConfigEnv() { os.Unsetenv("OPTIMUS_VERSION") os.Unsetenv("OPTIMUS_LOG_LEVEL") os.Unsetenv("OPTIMUS_SERVE_PORT") From 4c37b2c381280915093dada3c5b7a6df25e7a434 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 25 Mar 2022 10:21:36 +0700 Subject: [PATCH 27/54] fix: use config on version command only when flag with-server is true --- cmd/version.go | 24 +++++++++++++----------- scripts/smoke-test.sh | 2 +- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index fcc0aa1016..f4003a5d10 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -32,24 +32,26 @@ func versionCommand(pluginRepo models.PluginRepository) *cli.Command { c.Flags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") c.RunE = func(c *cli.Command, args []string) error { - // TODO: find a way to load the config in one place - conf, err := config.LoadClientConfig(configFilePath) - if err != nil { - return err - } - - if err := config.Validate(conf); err != nil { - return err - } - // initiate logger - l := initLogger(plainLoggerType, conf.Log) + l := initLogger(plainLoggerType, config.LogConfig{}) // Print client version l.Info(fmt.Sprintf("Client: %s-%s", coloredNotice(config.BuildVersion), coloredNotice(config.BuildCommit))) // Print server version if isWithServer { + // TODO: find a way to load the config in one place + conf, err := config.LoadClientConfig(configFilePath) + if err != nil { + return err + } + + if err := config.Validate(conf); err != nil { + return err + } + + l = initLogger(plainLoggerType, conf.Log) + srvVer, err := getVersionRequest(config.BuildVersion, conf.Host) if err != nil { return err diff --git a/scripts/smoke-test.sh b/scripts/smoke-test.sh index 8011d1e4ca..cac5e2ebb9 100644 --- a/scripts/smoke-test.sh +++ b/scripts/smoke-test.sh @@ -1,6 +1,6 @@ #!/bin/sh -./optimus version -c ./optimus.sample.yaml &> /dev/null +./optimus version &> /dev/null STATUS=$? if [ ! $STATUS -eq 0 ]; then echo "[smoke test] FAIL: optimus exited with code ${STATUS}"; From 0a75bb1474eac221be61356e389fa58fbfe23d3d Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 28 Mar 2022 10:47:56 +0700 Subject: [PATCH 28/54] fix: build with BuildVersion --- .goreleaser.yml | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.goreleaser.yml b/.goreleaser.yml index 915539d9d3..e3357f2058 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -10,7 +10,7 @@ builds: flags: - -a ldflags: - - -s -w -X github.com/odpf/optimus/config.Version={{.Tag}} -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}} + - -s -w -X github.com/odpf/optimus/config.BuildVersion={{.Tag}} -X github.com/odpf/optimus/config.BuildCommit={{.FullCommit}} -X github.com/odpf/optimus/config.BuildDate={{.Date}} goos: - linux - darwin diff --git a/Makefile b/Makefile index d13c2bfe54..52f1b58f4f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ PROTON_COMMIT := "f28898cb480769234c5d1e5540b94da1e5949407" build: # build optimus binary @echo " > notice: skipped proto generation, use 'generate-proto' make command" @echo " > building optimus version ${OPMS_VERSION}" - @go build -ldflags "-X ${NAME}/config.Version=${OPMS_VERSION} -X ${NAME}/config.BuildCommit=${LAST_COMMIT}" -o optimus . + @go build -ldflags "-X ${NAME}/config.BuildVersion=${OPMS_VERSION} -X ${NAME}/config.BuildCommit=${LAST_COMMIT}" -o optimus . @echo " - build complete" test-ci: smoke-test unit-test-ci vet ## run tests From a7084c14615ff982a0a2bbd99d918cfebf919b21 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 28 Mar 2022 12:28:26 +0700 Subject: [PATCH 29/54] refactor: replay struct --- cmd/server/server.go | 2 +- config/config_server.go | 20 ++++++++++++-------- config/loader_test.go | 36 ++++++++++++++++++++---------------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index db8d763a2a..3f69a55147 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -43,7 +43,7 @@ func checkRequiredConfigs(conf config.Serve) error { if conf.IngressHost == "" { return fmt.Errorf("serve.ingress_host: %w", errRequiredMissing) } - if conf.ReplayNumWorkers < 1 { + if conf.Replay.NumWorkers < 1 { return fmt.Errorf("%s should be greater than 0", config.KeyServeReplayNumWorkers) } if conf.DB.DSN == "" { diff --git a/config/config_server.go b/config/config_server.go index 0e05d2c934..d6354e9b78 100644 --- a/config/config_server.go +++ b/config/config_server.go @@ -13,14 +13,18 @@ type ServerConfig struct { } type Serve struct { - Port int `mapstructure:"port" default:"9100"` // port to listen on - Host string `mapstructure:"host" default:"0.0.0.0"` // the network interface to listen on - IngressHost string `mapstructure:"ingress_host"` // service ingress host for jobs to communicate back to optimus - AppKey string `mapstructure:"app_key"` // random 32 character hash used for encrypting secrets - DB DBConfig `mapstructure:"db"` - ReplayNumWorkers int `mapstructure:"replay_num_workers" default:"1"` - ReplayWorkerTimeout time.Duration `mapstructure:"replay_worker_timeout" default:"120s"` - ReplayRunTimeout time.Duration `mapstructure:"replay_run_timeout"` + Port int `mapstructure:"port" default:"9100"` // port to listen on + Host string `mapstructure:"host" default:"0.0.0.0"` // the network interface to listen on + IngressHost string `mapstructure:"ingress_host"` // service ingress host for jobs to communicate back to optimus + AppKey string `mapstructure:"app_key"` // random 32 character hash used for encrypting secrets + DB DBConfig `mapstructure:"db"` + Replay Replay `mapstructure:"replay"` +} + +type Replay struct { + NumWorkers int `mapstructure:"num_workers" default:"1"` + WorkerTimeout time.Duration `mapstructure:"worker_timeout" default:"120s"` + RunTimeout time.Duration `mapstructure:"run_timeout"` } type DBConfig struct { diff --git a/config/loader_test.go b/config/loader_test.go index ed4b00d1bd..c2ce6bc01a 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -43,9 +43,10 @@ serve: host: localhost ingress_host: optimus.example.io:80 app_key: Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc - replay_num_workers: 1 - replay_worker_timeout: "100s" - replay_run_timeout: "10s" + replay: + num_workers: 1 + worker_timeout: "100s" + run_timeout: "10s" db: dsn: postgres://user:password@localhost:5432/database?sslmode=disable max_idle_connection: 5 @@ -81,9 +82,10 @@ serve: host: localhost ingress_host: optimus.example.io:80 app_key: Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc - replay_num_workers: 1 - replay_worker_timeout: "100s" - replay_run_timeout: "10s" + replay: + num_workers: 1 + worker_timeout: "100s" + run_timeout: "10s" db: dsn: postgres://user:password@localhost:5432/database?sslmode=disable max_idle_connection: 5 @@ -157,13 +159,15 @@ func TestLoadOptimusConfig(t *testing.T) { }, }, Server: config.Serve{ - Port: 9100, - Host: "localhost", - IngressHost: "optimus.example.io:80", - AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", - ReplayNumWorkers: 1, - ReplayWorkerTimeout: 100 * time.Second, - ReplayRunTimeout: 10 * time.Second, + Port: 9100, + Host: "localhost", + IngressHost: "optimus.example.io:80", + AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", + Replay: config.Replay{ + NumWorkers: 1, + WorkerTimeout: 100 * time.Second, + RunTimeout: 10 * time.Second, + }, DB: config.DBConfig{ DSN: "postgres://user:password@localhost:5432/database?sslmode=disable", MaxIdleConnection: 5, @@ -414,9 +418,9 @@ func (s *ConfigTestSuite) initExpectedServerConfig() { s.expectedServerConfig.Serve.Host = "localhost" s.expectedServerConfig.Serve.IngressHost = "optimus.example.io:80" s.expectedServerConfig.Serve.AppKey = "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc" - s.expectedServerConfig.Serve.ReplayNumWorkers = 1 - s.expectedServerConfig.Serve.ReplayWorkerTimeout = 100 * time.Second - s.expectedServerConfig.Serve.ReplayRunTimeout = 10 * time.Second + s.expectedServerConfig.Serve.Replay.NumWorkers = 1 + s.expectedServerConfig.Serve.Replay.WorkerTimeout = 100 * time.Second + s.expectedServerConfig.Serve.Replay.RunTimeout = 10 * time.Second s.expectedServerConfig.Serve.DB = config.DBConfig{} s.expectedServerConfig.Serve.DB.DSN = "postgres://user:password@localhost:5432/database?sslmode=disable" s.expectedServerConfig.Serve.DB.MaxIdleConnection = 5 From 2d125c472c25a6c4c3dfe09073b5a65362f34014 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 28 Mar 2022 12:30:51 +0700 Subject: [PATCH 30/54] refactor: put comment legacy for the deprecated function --- config/loader.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/loader.go b/config/loader.go index 31286cf9b8..3ac1eaa0aa 100644 --- a/config/loader.go +++ b/config/loader.go @@ -77,6 +77,7 @@ func LoadOptimusConfig(dirPaths ...string) (*Optimus, error) { return &optimus, nil } +// (LEGACY) func validateNamespaceDuplication(optimus *Optimus) error { nameToAppearance := make(map[string]int) for _, namespace := range optimus.Namespaces { @@ -94,6 +95,7 @@ func validateNamespaceDuplication(optimus *Optimus) error { return nil } +// (LEGACY) func loadConfig(cfg interface{}, fs afero.Fs, dirPath string) error { // getViperWithDefault + SetFs v := viper.New() From da6bda94a4fd14394b48bcf5384a8e44e2ea85c1 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 28 Mar 2022 12:32:37 +0700 Subject: [PATCH 31/54] fix: restore cli request fail --- main.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/main.go b/main.go index b6b08e4c3d..9f2ae3d504 100644 --- a/main.go +++ b/main.go @@ -1,6 +1,7 @@ package main import ( + "errors" "fmt" "math/rand" _ "net/http/pprof" @@ -12,6 +13,8 @@ import ( "github.com/odpf/optimus/models" ) +var errRequestFail = errors.New("🔥 unable to complete request successfully") + //nolint:forbidigo func main() { rand.Seed(time.Now().UTC().UnixNano()) @@ -22,7 +25,7 @@ func main() { ) if err := command.Execute(); err != nil { - fmt.Println(err) + fmt.Println(errRequestFail) os.Exit(1) } } From 9c10de3cf86156ef4296c16244a24e60624a215d Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 28 Mar 2022 12:43:52 +0700 Subject: [PATCH 32/54] refactor: pass global repo to relevant commands --- cmd/backup.go | 9 ++++----- cmd/backup_create.go | 10 ++++++---- cmd/backup_list.go | 5 +++-- cmd/backup_status.go | 5 +++-- cmd/commands.go | 12 ++++++------ cmd/deploy.go | 4 +++- cmd/job.go | 13 ++++++------- cmd/job_create.go | 3 ++- cmd/job_hook.go | 3 ++- cmd/job_render.go | 3 ++- cmd/job_run.go | 3 ++- cmd/job_validate.go | 3 ++- cmd/resource.go | 11 ++++++----- cmd/version.go | 3 ++- main.go | 6 +----- 15 files changed, 50 insertions(+), 43 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 6bad0853b4..1dc573cbde 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -5,7 +5,6 @@ import ( "github.com/MakeNowJust/heredoc" "github.com/odpf/optimus/config" - "github.com/odpf/optimus/models" cli "github.com/spf13/cobra" ) @@ -13,7 +12,7 @@ const ( backupTimeout = time.Minute * 15 ) -func backupCommand(datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} l := initLogger(plainLoggerType, conf.Log) @@ -42,8 +41,8 @@ func backupCommand(datastoreRepo models.DatastoreRepo) *cli.Command { return nil } - cmd.AddCommand(backupCreateCommand(l, conf, datastoreRepo)) - cmd.AddCommand(backupListCommand(l, conf, datastoreRepo)) - cmd.AddCommand(backupStatusCommand(l, conf, datastoreRepo)) + cmd.AddCommand(backupCreateCommand(l, conf)) + cmd.AddCommand(backupListCommand(l, conf)) + cmd.AddCommand(backupStatusCommand(l, conf)) return cmd } diff --git a/cmd/backup_create.go b/cmd/backup_create.go index e8f9e4ae89..242ffb9965 100644 --- a/cmd/backup_create.go +++ b/cmd/backup_create.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupCreateCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupCreateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( backupCmd = &cli.Command{ Use: "create", @@ -42,11 +42,13 @@ func backupCreateCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo backupCmd.Flags().BoolVar(&ignoreDownstream, "ignore-downstream", ignoreDownstream, "Do not take backups for dependent downstream resources") backupCmd.RunE = func(cmd *cli.Command, args []string) error { + var err error + dsRepo := models.DatastoreRegistry namespace, err := askToSelectNamespace(l, conf) if err != nil { return err } - if storerName, err = extractDatastoreName(datastoreRepo, storerName); err != nil { + if storerName, err = extractDatastoreName(dsRepo, storerName); err != nil { return err } if resourceName, err = extractResourceName(resourceName); err != nil { @@ -115,9 +117,9 @@ func backupCreateCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo return backupCmd } -func extractDatastoreName(datastoreRepo models.DatastoreRepo, storerName string) (string, error) { +func extractDatastoreName(dsRepo models.DatastoreRepo, storerName string) (string, error) { availableStorer := []string{} - for _, s := range datastoreRepo.GetAll() { + for _, s := range dsRepo.GetAll() { availableStorer = append(availableStorer, s.Name()) } if storerName == "" { diff --git a/cmd/backup_list.go b/cmd/backup_list.go index e7396acb98..4e7434119f 100644 --- a/cmd/backup_list.go +++ b/cmd/backup_list.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupListCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupListCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( backupCmd = &cli.Command{ Use: "list", @@ -27,8 +27,9 @@ func backupListCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo mo ) backupCmd.Flags().StringVarP(&project, "project", "p", conf.Project.Name, "project name of optimus managed repository") backupCmd.RunE = func(cmd *cli.Command, args []string) error { + dsRepo := models.DatastoreRegistry availableStorer := []string{} - for _, s := range datastoreRepo.GetAll() { + for _, s := range dsRepo.GetAll() { availableStorer = append(availableStorer, s.Name()) } var storerName string diff --git a/cmd/backup_status.go b/cmd/backup_status.go index b70ada9dac..a895de1422 100644 --- a/cmd/backup_status.go +++ b/cmd/backup_status.go @@ -17,7 +17,7 @@ import ( "github.com/odpf/optimus/models" ) -func backupStatusCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func backupStatusCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( project string backupCmd = &cli.Command{ @@ -28,8 +28,9 @@ func backupStatusCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo ) backupCmd.Flags().StringVarP(&project, "project", "p", conf.Project.Name, "Project name of optimus managed repository") backupCmd.RunE = func(cmd *cli.Command, args []string) error { + dsRepo := models.DatastoreRegistry availableStorer := []string{} - for _, s := range datastoreRepo.GetAll() { + for _, s := range dsRepo.GetAll() { availableStorer = append(availableStorer, s.Name()) } var storerName string diff --git a/cmd/commands.go b/cmd/commands.go index fc6aa20a02..5549f1b0a3 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -58,7 +58,7 @@ type JobSpecRepository interface { // default output of logging should go to stdout // interactive output like progress bars should go to stderr // unless the stdout/err is a tty, colors/progressbar should be disabled -func New(pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { +func New() *cli.Command { disableColoredOut = !isTerminal(os.Stdout) cmd := &cli.Command{ @@ -108,13 +108,13 @@ func New(pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.C cmdx.SetHelp(cmd) cmd.PersistentFlags().BoolVar(&disableColoredOut, "no-color", disableColoredOut, "Disable colored output") - cmd.AddCommand(versionCommand(pluginRepo)) + cmd.AddCommand(versionCommand()) cmd.AddCommand(configCommand()) - cmd.AddCommand(jobCommand(pluginRepo)) - cmd.AddCommand(deployCommand(pluginRepo, dsRepo)) - cmd.AddCommand(resourceCommand(dsRepo)) + cmd.AddCommand(jobCommand()) + cmd.AddCommand(deployCommand()) + cmd.AddCommand(resourceCommand()) cmd.AddCommand(replayCommand()) - cmd.AddCommand(backupCommand(dsRepo)) + cmd.AddCommand(backupCommand()) cmd.AddCommand(adminCommand()) cmd.AddCommand(secretCommand()) diff --git a/cmd/deploy.go b/cmd/deploy.go index 55ef326410..98c3d3b59a 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -27,7 +27,7 @@ const ( ) // deployCommand pushes current repo to optimus service -func deployCommand(pluginRepo models.PluginRepository, dsRepo models.DatastoreRepo) *cli.Command { +func deployCommand() *cli.Command { var ( namespaces []string ignoreJobs bool @@ -54,6 +54,8 @@ func deployCommand(pluginRepo models.PluginRepository, dsRepo models.DatastoreRe cmd.Flags().BoolVar(&ignoreResources, "ignore-resources", false, "Ignore deployment of resources") cmd.RunE = func(c *cli.Command, args []string) error { + pluginRepo := models.PluginRegistry + dsRepo := models.DatastoreRegistry // TODO: find a way to load the config in one place conf, err := config.LoadClientConfig(configFilePath) if err != nil { diff --git a/cmd/job.go b/cmd/job.go index 2e11263211..a7a89aeefb 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -2,11 +2,10 @@ package cmd import ( "github.com/odpf/optimus/config" - "github.com/odpf/optimus/models" cli "github.com/spf13/cobra" ) -func jobCommand(pluginRepo models.PluginRepository) *cli.Command { +func jobCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} l := initLogger(plainLoggerType, conf.Log) @@ -34,11 +33,11 @@ func jobCommand(pluginRepo models.PluginRepository) *cli.Command { cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") - cmd.AddCommand(jobCreateCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobAddHookCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobRenderTemplateCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobValidateCommand(l, conf, pluginRepo)) - cmd.AddCommand(jobRunCommand(l, conf, pluginRepo)) + cmd.AddCommand(jobCreateCommand(l, conf)) + cmd.AddCommand(jobAddHookCommand(l, conf)) + cmd.AddCommand(jobRenderTemplateCommand(l, conf)) + cmd.AddCommand(jobValidateCommand(l, conf)) + cmd.AddCommand(jobRunCommand(l, conf)) cmd.AddCommand(jobRunListCommand(l, conf.Project.Name, conf.Host)) return cmd } diff --git a/cmd/job_create.go b/cmd/job_create.go index a2922e095f..e43eb07035 100644 --- a/cmd/job_create.go +++ b/cmd/job_create.go @@ -33,12 +33,13 @@ var ( specFileNames = []string{local.ResourceSpecFileName, local.JobSpecFileName} ) -func jobCreateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobCreateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new Job", Example: "optimus job create", RunE: func(cmd *cli.Command, args []string) error { + pluginRepo := models.PluginRegistry namespace, err := askToSelectNamespace(l, conf) if err != nil { return err diff --git a/cmd/job_hook.go b/cmd/job_hook.go index 7d1afd19fc..a7020d76df 100644 --- a/cmd/job_hook.go +++ b/cmd/job_hook.go @@ -16,7 +16,7 @@ import ( "github.com/odpf/optimus/utils" ) -func jobAddHookCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobAddHookCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { cmd := &cli.Command{ Use: "addhook", Aliases: []string{"add_hook", "add-hook", "addHook", "attach_hook", "attach-hook", "attachHook"}, @@ -24,6 +24,7 @@ func jobAddHookCommand(l log.Logger, conf *config.ClientConfig, pluginRepo model Long: "Add a runnable instance that will be triggered before or after the base transformation.", Example: "optimus addhook", RunE: func(cmd *cli.Command, args []string) error { + pluginRepo := models.PluginRegistry namespace, err := askToSelectNamespace(l, conf) if err != nil { return err diff --git a/cmd/job_render.go b/cmd/job_render.go index 0634dcefb9..8bbc0fcfd9 100644 --- a/cmd/job_render.go +++ b/cmd/job_render.go @@ -17,13 +17,14 @@ import ( "github.com/odpf/optimus/utils" ) -func jobRenderTemplateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobRenderTemplateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { cmd := &cli.Command{ Use: "render", Short: "Apply template values in job specification to current 'render' directory", Long: "Process optimus job specification based on macros/functions used.", Example: "optimus job render []", RunE: func(c *cli.Command, args []string) error { + pluginRepo := models.PluginRegistry namespace, err := askToSelectNamespace(l, conf) if err != nil { return err diff --git a/cmd/job_run.go b/cmd/job_run.go index 923f631af4..94947d7fa4 100644 --- a/cmd/job_run.go +++ b/cmd/job_run.go @@ -22,7 +22,7 @@ const ( runJobTimeout = time.Minute * 1 ) -func jobRunCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobRunCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( namespaceName string projectName = conf.Project.Name @@ -36,6 +36,7 @@ func jobRunCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.Pl Example: "optimus job run ", Hidden: true, RunE: func(c *cli.Command, args []string) error { + pluginRepo := models.PluginRegistry namespace, err := conf.GetNamespaceByName(namespaceName) if err != nil { return err diff --git a/cmd/job_validate.go b/cmd/job_validate.go index a2e19c216f..cc7959c971 100644 --- a/cmd/job_validate.go +++ b/cmd/job_validate.go @@ -23,7 +23,7 @@ const ( validateTimeout = time.Minute * 5 ) -func jobValidateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo models.PluginRepository) *cli.Command { +func jobValidateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { var ( verbose bool namespaceName string @@ -36,6 +36,7 @@ func jobValidateCommand(l log.Logger, conf *config.ClientConfig, pluginRepo mode Long: "Check if specifications are valid for deployment", Example: "optimus job validate", RunE: func(c *cli.Command, args []string) error { + pluginRepo := models.PluginRegistry namespace, err := conf.GetNamespaceByName(namespaceName) if err != nil { return err diff --git a/cmd/resource.go b/cmd/resource.go index bc3018ea2e..9e9bbbd317 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -23,7 +23,7 @@ import ( var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-zA-Z0-9_\-\.]+$`, `invalid name (can only contain characters A-Z (in either case), 0-9, "-", "_" or "." and must start with an alphanumeric character)`) -func resourceCommand(datastoreRepo models.DatastoreRepo) *cli.Command { +func resourceCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} l := initLogger(plainLoggerType, conf.Log) @@ -48,11 +48,11 @@ func resourceCommand(datastoreRepo models.DatastoreRepo) *cli.Command { return nil } - cmd.AddCommand(createResourceSubCommand(l, conf, datastoreRepo)) + cmd.AddCommand(createResourceSubCommand(l, conf)) return cmd } -func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastoreRepo models.DatastoreRepo) *cli.Command { +func createResourceSubCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { cmd := &cli.Command{ Use: "create", Short: "Create a new resource", @@ -60,6 +60,7 @@ func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastore } cmd.RunE = func(cmd *cli.Command, args []string) error { + dsRepo := models.DatastoreRegistry // init local specs datastoreSpecFs := make(map[string]map[string]afero.Fs) for _, namespace := range conf.Namespaces { @@ -75,7 +76,7 @@ func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastore return err } availableStorer := []string{} - for _, s := range datastoreRepo.GetAll() { + for _, s := range dsRepo.GetAll() { availableStorer = append(availableStorer, s.Name()) } var storerName string @@ -92,7 +93,7 @@ func createResourceSubCommand(l log.Logger, conf *config.ClientConfig, datastore // find requested datastore availableTypes := []string{} - datastore, _ := datastoreRepo.GetByName(storerName) + datastore, _ := dsRepo.GetByName(storerName) for dsType := range datastore.Types() { availableTypes = append(availableTypes, dsType.String()) } diff --git a/cmd/version.go b/cmd/version.go index f4003a5d10..adfa820c37 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -18,7 +18,7 @@ const ( githubRepo = "odpf/optimus" ) -func versionCommand(pluginRepo models.PluginRepository) *cli.Command { +func versionCommand() *cli.Command { var isWithServer bool var configFilePath string @@ -66,6 +66,7 @@ func versionCommand(pluginRepo models.PluginRepository) *cli.Command { } // Print all plugin infos + pluginRepo := models.PluginRegistry plugins := pluginRepo.GetAll() l.Info(fmt.Sprintf("\nDiscovered plugins: %d", len(plugins))) for taskIdx, tasks := range plugins { diff --git a/main.go b/main.go index 9f2ae3d504..4bb0487758 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,6 @@ import ( "github.com/odpf/optimus/cmd" _ "github.com/odpf/optimus/ext/datastore" - "github.com/odpf/optimus/models" ) var errRequestFail = errors.New("🔥 unable to complete request successfully") @@ -19,10 +18,7 @@ var errRequestFail = errors.New("🔥 unable to complete request successfully") func main() { rand.Seed(time.Now().UTC().UnixNano()) - command := cmd.New( - models.PluginRegistry, - models.DatastoreRegistry, - ) + command := cmd.New() if err := command.Execute(); err != nil { fmt.Println(errRequestFail) From 45a837f37c94f6cc4c8bd6ba24ee8812ba552f91 Mon Sep 17 00:00:00 2001 From: Anwar Hidayat Date: Mon, 28 Mar 2022 14:49:44 +0700 Subject: [PATCH 33/54] refactor: extract datastore initialization to a function --- cmd/commands.go | 14 ++++++++++++++ cmd/deploy.go | 11 +---------- cmd/resource.go | 16 +++------------- 3 files changed, 18 insertions(+), 23 deletions(-) diff --git a/cmd/commands.go b/cmd/commands.go index 5549f1b0a3..d124193c63 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -13,9 +13,11 @@ import ( grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/mattn/go-isatty" + "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" "github.com/odpf/salt/cmdx" "github.com/odpf/salt/term" + "github.com/spf13/afero" cli "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" @@ -193,3 +195,15 @@ func (a *BasicAuthentication) RequireTransportSecurity() bool { func isTerminal(f *os.File) bool { return isatty.IsTerminal(f.Fd()) || isatty.IsCygwinTerminal(f.Fd()) } + +func getDatastoreSpecFs(namespaces []*config.Namespace) map[string]map[string]afero.Fs { + output := make(map[string]map[string]afero.Fs) + for _, namespace := range namespaces { + dtSpec := make(map[string]afero.Fs) + for _, dsConfig := range namespace.Datastore { + dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) + } + output[namespace.Name] = dtSpec + } + return output +} diff --git a/cmd/deploy.go b/cmd/deploy.go index 98c3d3b59a..6d8a8f86ff 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -62,18 +62,9 @@ func deployCommand() *cli.Command { return err } - // init logger l := initLogger(plainLoggerType, conf.Log) + datastoreSpecFs := getDatastoreSpecFs(conf.Namespaces) - // init local specs - datastoreSpecFs := make(map[string]map[string]afero.Fs) - for _, namespace := range conf.Namespaces { - dtSpec := make(map[string]afero.Fs) - for _, dsConfig := range namespace.Datastore { - dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) - } - datastoreSpecFs[namespace.Name] = dtSpec - } l.Info(fmt.Sprintf("Deploying project: %s to %s", conf.Project.Name, conf.Host)) start := time.Now() diff --git a/cmd/resource.go b/cmd/resource.go index 9e9bbbd317..397e61cfaa 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -9,15 +9,13 @@ import ( "strings" "github.com/AlecAivazis/survey/v2" - "github.com/odpf/salt/log" - "github.com/spf13/afero" - cli "github.com/spf13/cobra" - "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" "github.com/odpf/optimus/store" "github.com/odpf/optimus/store/local" "github.com/odpf/optimus/utils" + "github.com/odpf/salt/log" + cli "github.com/spf13/cobra" ) var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-zA-Z0-9_\-\.]+$`, @@ -61,15 +59,7 @@ func createResourceSubCommand(l log.Logger, conf *config.ClientConfig) *cli.Comm cmd.RunE = func(cmd *cli.Command, args []string) error { dsRepo := models.DatastoreRegistry - // init local specs - datastoreSpecFs := make(map[string]map[string]afero.Fs) - for _, namespace := range conf.Namespaces { - dtSpec := make(map[string]afero.Fs) - for _, dsConfig := range namespace.Datastore { - dtSpec[dsConfig.Type] = afero.NewBasePathFs(afero.NewOsFs(), dsConfig.Path) - } - datastoreSpecFs[namespace.Name] = dtSpec - } + datastoreSpecFs := getDatastoreSpecFs(conf.Namespaces) namespace, err := askToSelectNamespace(l, conf) if err != nil { From 29e3b988e4a8c20404f06f9bfac2653459f53a3a Mon Sep 17 00:00:00 2001 From: Anwar Hidayat Date: Mon, 28 Mar 2022 14:57:53 +0700 Subject: [PATCH 34/54] refactor: clean-up variable naming --- cmd/config.go | 3 ++- config/loader_test.go | 8 ++++---- optimus.sample.yaml | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/config.go b/cmd/config.go index 7817b167b1..1385d48128 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -92,7 +92,8 @@ func configInitCommand() *cli.Command { if err != nil { return err } - if err := ioutil.WriteFile(fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension), confMarshaled, defaultFilePermissionMode); err != nil { + filePath := fmt.Sprintf("%s.%s", config.DefaultFilename, config.DefaultFileExtension) + if err := ioutil.WriteFile(filePath, confMarshaled, defaultFilePermissionMode); err != nil { return err } diff --git a/config/loader_test.go b/config/loader_test.go index c2ce6bc01a..a2eb15e226 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/suite" ) -const projectConfig = ` +const clientConfig = ` version: 1 log: level: info @@ -253,7 +253,7 @@ func TestConfig(t *testing.T) { func (s *ConfigTestSuite) TestLoadClientConfig() { currFilePath := path.Join(s.currPath, config.DefaultFilename+"."+config.DefaultFileExtension) - s.a.WriteFile(currFilePath, []byte(projectConfig), fs.ModeTemporary) + s.a.WriteFile(currFilePath, []byte(clientConfig), fs.ModeTemporary) s.Run("WhenFilepathIsEmpty", func() { s.Run("WhenConfigInCurrentPathIsExist", func() { @@ -266,7 +266,7 @@ func (s *ConfigTestSuite) TestLoadClientConfig() { s.Run("WhenConfigInCurrentPathNotExist", func() { s.a.Remove(currFilePath) - defer s.a.WriteFile(currFilePath, []byte(projectConfig), fs.ModeTemporary) + defer s.a.WriteFile(currFilePath, []byte(clientConfig), fs.ModeTemporary) conf, err := config.LoadClientConfig(config.EmptyPath) s.Assert().NoError(err) @@ -278,7 +278,7 @@ func (s *ConfigTestSuite) TestLoadClientConfig() { s.Run("WhenFilePathIsvalid", func() { samplePath := "./sample/path/config.yaml" b := strings.Builder{} - b.WriteString(projectConfig) + b.WriteString(clientConfig) b.WriteString(`- name: namespace-c job: path: ./jobs-c diff --git a/optimus.sample.yaml b/optimus.sample.yaml index 6de14f6803..69d1b4abbd 100644 --- a/optimus.sample.yaml +++ b/optimus.sample.yaml @@ -1,5 +1,5 @@ ######################################## -# PROJECT CONFIG +# CLIENT CONFIG ######################################## version: 1 From 940ef445b3e186c620bda8d78bf15d247a0fcd23 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 29 Mar 2022 11:32:48 +0700 Subject: [PATCH 35/54] fix: use replay struct --- cmd/server/optimus.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/server/optimus.go b/cmd/server/optimus.go index f0855cf04e..eb11c165c8 100644 --- a/cmd/server/optimus.go +++ b/cmd/server/optimus.go @@ -215,9 +215,9 @@ func (s *OptimusServer) setupHandlers() error { ) replayManager := job.NewManager(s.logger, replayWorkerFactory, replaySpecRepoFac, utils.NewUUIDProvider(), job.ReplayManagerConfig{ - NumWorkers: s.conf.Serve.ReplayNumWorkers, - WorkerTimeout: s.conf.Serve.ReplayWorkerTimeout, - RunTimeout: s.conf.Serve.ReplayRunTimeout, + NumWorkers: s.conf.Serve.Replay.NumWorkers, + WorkerTimeout: s.conf.Serve.Replay.WorkerTimeout, + RunTimeout: s.conf.Serve.Replay.RunTimeout, }, scheduler, replayValidator, replaySyncer) notificationContext, cancelNotifiers := context.WithCancel(context.Background()) From 0e0ce74b0d62d2416fe0c94f7afc609d354760c9 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 29 Mar 2022 11:35:16 +0700 Subject: [PATCH 36/54] fix: accpeting config file path on server command --- cmd/serve.go | 95 +++++++++++++++++++++++++++------------------------- 1 file changed, 49 insertions(+), 46 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 5daed7d8cc..65cb8d19c4 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -15,62 +15,65 @@ import ( ) func serveCommand() *cli.Command { - c := &cli.Command{ + var configFilePath string + cmd := &cli.Command{ Use: "serve", Short: "Starts optimus service", Example: "optimus serve", Annotations: map[string]string{ "group:other": "dev", }, - RunE: func(c *cli.Command, args []string) error { - // TODO: find a way to load the config in one place - conf, err := config.LoadServerConfig() - if err != nil { - panic(err.Error()) - return nil - } + } - // initiate jsonLogger - l := initLogger(jsonLoggerType, conf.Log) - pluginLogLevel := hclog.Info - if conf.Log.Level == config.LogLevelDebug { - pluginLogLevel = hclog.Debug - } + cmd.Flags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") - // discover and load plugins. TODO: refactor this - if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ - Name: "optimus", - Output: os.Stdout, - Level: pluginLogLevel, - })); err != nil { - hPlugin.CleanupClients() - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(1) - } - // Make sure we clean up any managed plugins at the end of this - defer hPlugin.CleanupClients() + cmd.RunE = func(c *cli.Command, args []string) error { + // TODO: find a way to load the config in one place + conf, err := config.LoadServerConfig(configFilePath) + if err != nil { + return err + } - // init telemetry - teleShutdown, err := config.InitTelemetry(l, conf.Telemetry) - if err != nil { - fmt.Printf("ERROR: %s\n", err.Error()) - os.Exit(1) - } - defer teleShutdown() - l.Info(coloredSuccess("Starting Optimus"), "version", config.BuildVersion) - optimusServer, err := server.New(l, *conf) - defer optimusServer.Shutdown() - if err != nil { - return fmt.Errorf("unable to create server: %w", err) - } + // initiate jsonLogger + l := initLogger(jsonLoggerType, conf.Log) + pluginLogLevel := hclog.Info + if conf.Log.Level == config.LogLevelDebug { + pluginLogLevel = hclog.Debug + } - sigc := make(chan os.Signal, 1) - signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) - <-sigc - l.Info(coloredNotice("Shutting down server")) - return nil - }, + // discover and load plugins. TODO: refactor this + if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ + Name: "optimus", + Output: os.Stdout, + Level: pluginLogLevel, + })); err != nil { + hPlugin.CleanupClients() + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + // Make sure we clean up any managed plugins at the end of this + defer hPlugin.CleanupClients() + + // init telemetry + teleShutdown, err := config.InitTelemetry(l, conf.Telemetry) + if err != nil { + fmt.Printf("ERROR: %s\n", err.Error()) + os.Exit(1) + } + defer teleShutdown() + l.Info(coloredSuccess("Starting Optimus"), "version", config.BuildVersion) + optimusServer, err := server.New(l, *conf) + defer optimusServer.Shutdown() + if err != nil { + return fmt.Errorf("unable to create server: %w", err) + } + + sigc := make(chan os.Signal, 1) + signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) + <-sigc + l.Info(coloredNotice("Shutting down server")) + return nil } - return c + return cmd } From 23b3137e1e73e23fb3c17c4c1cc739a95bbc5c27 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 29 Mar 2022 11:41:11 +0700 Subject: [PATCH 37/54] refactor: clean up legacy loader --- config/loader.go | 70 -------------------- config/loader_test.go | 147 ------------------------------------------ 2 files changed, 217 deletions(-) diff --git a/config/loader.go b/config/loader.go index 3ac1eaa0aa..bffd2229cd 100644 --- a/config/loader.go +++ b/config/loader.go @@ -1,7 +1,6 @@ package config import ( - "errors" "fmt" "os" "strings" @@ -48,75 +47,6 @@ func init() { homePath = p } -// LoadOptimusConfig Load configuration file from following paths (LEGACY) -// ./ -// / -// ~/.optimus/ -// Namespaces will be loaded only from current project ./ -func LoadOptimusConfig(dirPaths ...string) (*Optimus, error) { - fs := afero.NewReadOnlyFs(afero.NewOsFs()) - - var targetPath string - if len(dirPaths) > 0 { - targetPath = dirPaths[0] - } else { - currPath, err := os.Getwd() - if err != nil { - return nil, fmt.Errorf("error getting current work directory path: %w", err) - } - targetPath = currPath - } - - optimus := Optimus{} - if err := loadConfig(&optimus, fs, targetPath); err != nil { - return nil, errors.New("error loading config") - } - if err := validateNamespaceDuplication(&optimus); err != nil { - return nil, err - } - return &optimus, nil -} - -// (LEGACY) -func validateNamespaceDuplication(optimus *Optimus) error { - nameToAppearance := make(map[string]int) - for _, namespace := range optimus.Namespaces { - nameToAppearance[namespace.Name]++ - } - var duplicateNames []string - for name, appearance := range nameToAppearance { - if appearance > 1 { - duplicateNames = append(duplicateNames, name) - } - } - if len(duplicateNames) > 0 { - return fmt.Errorf("namespaces [%s] are duplicate", strings.Join(duplicateNames, ", ")) - } - return nil -} - -// (LEGACY) -func loadConfig(cfg interface{}, fs afero.Fs, dirPath string) error { - // getViperWithDefault + SetFs - v := viper.New() - v.SetConfigName("config") - v.SetConfigType("yaml") - v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) - v.SetFs(fs) - - opts := []config.LoaderOption{ - config.WithViper(v), - config.WithName(".optimus"), - config.WithType(DefaultFileExtension), - config.WithEnvPrefix("OPTIMUS"), - config.WithEnvKeyReplacer(".", "_"), - config.WithPath(dirPath), - } - - l := config.NewLoader(opts...) - return l.Load(cfg) -} - // LoadClientConfig load the project specific config from these locations: // 1. filepath. ./optimus -c "path/to/config/optimus.yaml" // 2. current dir. Optimus will look at current directory if there's optimus.yaml there, use it diff --git a/config/loader_test.go b/config/loader_test.go index a2eb15e226..621aa2934e 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -10,7 +10,6 @@ import ( "github.com/odpf/optimus/config" "github.com/spf13/afero" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) @@ -59,152 +58,6 @@ telemetry: jaeger_addr: "http://localhost:14268/api/traces" ` -// (LEGACY) -const ( - configFileName = ".optimus.yaml" - optimusConfigDirName = "./optimus" -) - -// (LEGACY) -const optimusConfigContent = ` -version: 1 -log: - level: info -host: "localhost:9100" -project: - name: sample_project - config: - environment: integration - scheduler_host: http://example.io/ - storage_path: file://absolute_path_to_a_directory -serve: - port: 9100 - host: localhost - ingress_host: optimus.example.io:80 - app_key: Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc - replay: - num_workers: 1 - worker_timeout: "100s" - run_timeout: "10s" - db: - dsn: postgres://user:password@localhost:5432/database?sslmode=disable - max_idle_connection: 5 - max_open_connection: 10 -scheduler: - name: airflow2 - skip_init: true -telemetry: - profile_addr: ":9110" - jaeger_addr: "http://localhost:14268/api/traces" -namespaces: -- name: namespace-a - job: - path: ./jobs-a -- name: namespace-b - job: - path: ./jobs-b -` - -// (LEGACY) -func setup(content string) { - teardown() - if err := os.Mkdir(optimusConfigDirName, 0o750); err != nil { - panic(err) - } - confPath := path.Join(optimusConfigDirName, configFileName) - if err := os.WriteFile(confPath, []byte(content), 0o660); err != nil { - panic(err) - } - unsetServerConfigEnv() -} - -func teardown() { - if err := os.RemoveAll(optimusConfigDirName); err != nil { - panic(err) - } -} - -// (LEGACY) -func TestLoadOptimusConfig(t *testing.T) { - t.Run("should return config and nil if no error is found", func(t *testing.T) { - setup(optimusConfigContent + `- name: namespace-b - job: - path: ./jobs-b`) - defer teardown() - - expectedErrMsg := "namespaces [namespace-b] are duplicate" - - actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) - - assert.Nil(t, actualConf) - assert.EqualError(t, actualErr, expectedErrMsg) - }) - - t.Run("should return config and nil if no error is found", func(t *testing.T) { - setup(optimusConfigContent) - defer teardown() - - expectedConf := &config.Optimus{ - Version: 1, - Log: config.LogConfig{ - Level: "info", - }, - Host: "localhost:9100", - Project: config.Project{ - Name: "sample_project", - Config: map[string]string{ - "environment": "integration", - "scheduler_host": "http://example.io/", - "storage_path": "file://absolute_path_to_a_directory", - }, - }, - Server: config.Serve{ - Port: 9100, - Host: "localhost", - IngressHost: "optimus.example.io:80", - AppKey: "Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc", - Replay: config.Replay{ - NumWorkers: 1, - WorkerTimeout: 100 * time.Second, - RunTimeout: 10 * time.Second, - }, - DB: config.DBConfig{ - DSN: "postgres://user:password@localhost:5432/database?sslmode=disable", - MaxIdleConnection: 5, - MaxOpenConnection: 10, - }, - }, - Scheduler: config.SchedulerConfig{ - Name: "airflow2", - SkipInit: true, - }, - Telemetry: config.TelemetryConfig{ - ProfileAddr: ":9110", - JaegerAddr: "http://localhost:14268/api/traces", - }, - Namespaces: []*config.Namespace{ - { - Name: "namespace-a", - Job: config.Job{ - Path: "./jobs-a", - }, - }, - { - Name: "namespace-b", - Job: config.Job{ - Path: "./jobs-b", - }, - }, - }, - } - - actualConf, actualErr := config.LoadOptimusConfig(optimusConfigDirName) - - assert.EqualValues(t, expectedConf, actualConf) - assert.NoError(t, actualErr) - }) -} - type ConfigTestSuite struct { suite.Suite a afero.Afero From 380f3611bf25734cfde694512a5097a0cdb8d4d6 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 29 Mar 2022 11:46:27 +0700 Subject: [PATCH 38/54] fix: remove server config load from home dir --- config/loader.go | 18 +++++++++--------- config/loader_test.go | 25 ++----------------------- 2 files changed, 11 insertions(+), 32 deletions(-) diff --git a/config/loader.go b/config/loader.go index bffd2229cd..e7326db31f 100644 --- a/config/loader.go +++ b/config/loader.go @@ -12,11 +12,12 @@ import ( ) const ( - ErrFailedToRead = "unable to read optimus config file %v (%s)" - DefaultFilename = "optimus" - DefaultFileExtension = "yaml" - DefaultEnvPrefix = "OPTIMUS" - EmptyPath = "" + ErrFailedToRead = "unable to read optimus config file %v (%s)" + DefaultFilename = "optimus" + DefaultConfigFilename = "config" // default file name for server config + DefaultFileExtension = "yaml" + DefaultEnvPrefix = "OPTIMUS" + EmptyPath = "" ) var ( @@ -89,7 +90,6 @@ func LoadClientConfig(filePath string) (*ClientConfig, error) { // 1. filepath. ./optimus -c "path/to/config.yaml" // 2. env var. eg. OPTIMUS_SERVE_PORT, etc // 3. executable binary location -// 4. home dir func LoadServerConfig(filePath string) (*ServerConfig, error) { cfg := &ServerConfig{} @@ -99,7 +99,7 @@ func LoadServerConfig(filePath string) (*ServerConfig, error) { opts := []config.LoaderOption{ config.WithViper(v), - config.WithName(DefaultFilename), + config.WithName(DefaultConfigFilename), config.WithType(DefaultFileExtension), } @@ -113,8 +113,8 @@ func LoadServerConfig(filePath string) (*ServerConfig, error) { // load opt from env var opts = append(opts, config.WithEnvPrefix(DefaultEnvPrefix), config.WithEnvKeyReplacer(".", "_")) - // load opt from exec & home directory - opts = append(opts, config.WithPath(execPath), config.WithPath(homePath)) + // load opt from exec + opts = append(opts, config.WithPath(execPath)) } // load the config diff --git a/config/loader_test.go b/config/loader_test.go index 621aa2934e..5fd840c25e 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -63,7 +63,6 @@ type ConfigTestSuite struct { a afero.Afero currPath string execPath string - homePath string expectedClientConfig *config.ClientConfig expectedServerConfig *config.ServerConfig @@ -83,11 +82,6 @@ func (s *ConfigTestSuite) SetupTest() { s.execPath = p s.a.Fs.MkdirAll(s.execPath, fs.ModeTemporary) - p, err = os.UserHomeDir() - s.Require().NoError(err) - s.homePath = p - s.a.Fs.MkdirAll(s.homePath, fs.ModeTemporary) - config.FS = s.a.Fs s.initExpectedClientConfig() @@ -97,7 +91,6 @@ func (s *ConfigTestSuite) SetupTest() { func (s *ConfigTestSuite) TearDownTest() { s.a.Fs.RemoveAll(s.currPath) s.a.Fs.RemoveAll(s.execPath) - s.a.Fs.RemoveAll(s.homePath) } func TestConfig(t *testing.T) { @@ -156,10 +149,8 @@ func (s *ConfigTestSuite) TestLoadClientConfig() { } func (s *ConfigTestSuite) TestLoadServerConfig() { - execFilePath := path.Join(s.execPath, config.DefaultFilename+"."+config.DefaultFileExtension) - homeFilePath := path.Join(s.homePath, config.DefaultFilename+"."+config.DefaultFileExtension) + execFilePath := path.Join(s.execPath, config.DefaultConfigFilename+"."+config.DefaultFileExtension) s.a.WriteFile(execFilePath, []byte(serverConfig), fs.ModeTemporary) - s.a.WriteFile(homeFilePath, []byte(`version: 3`), fs.ModeTemporary) s.initServerConfigEnv() s.Run("WhenFilepathIsEmpty", func() { @@ -193,19 +184,7 @@ func (s *ConfigTestSuite) TestLoadServerConfig() { s.Assert().NoError(err) s.Assert().NotNil(conf) - s.Assert().Equal("3", conf.Version.String()) // should load from home dir - s.Assert().Equal("INFO", conf.Log.Level.String()) - }) - - s.Run("WhenConfigNotFound", func() { - s.a.Remove(execFilePath) - s.a.Remove(homeFilePath) - defer s.a.WriteFile(execFilePath, []byte(serverConfig), fs.ModeTemporary) - defer s.a.WriteFile(homeFilePath, []byte(`version: 3`), fs.ModeTemporary) - - conf, err := config.LoadServerConfig(config.EmptyPath) - s.Assert().NoError(err) - s.Assert().NotNil(conf) + s.Assert().Equal("0", conf.Version.String()) }) }) From 7c9869c71ad7d4d3b2ca77c016c2f98a126863a6 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Tue, 29 Mar 2022 11:55:57 +0700 Subject: [PATCH 39/54] fix: linter --- cmd/backup.go | 3 ++- cmd/commands.go | 5 +++-- cmd/config.go | 3 ++- cmd/extension.go | 3 ++- cmd/job.go | 3 ++- cmd/logger.go | 3 ++- cmd/replay.go | 3 ++- cmd/resource.go | 5 +++-- cmd/serve.go | 7 ++++--- cmd/version.go | 7 ++++--- config/loader.go | 8 -------- config/loader_test.go | 3 ++- config/validation_test.go | 3 ++- main.go | 1 - 14 files changed, 30 insertions(+), 27 deletions(-) diff --git a/cmd/backup.go b/cmd/backup.go index 1dc573cbde..faa8c324ad 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -4,8 +4,9 @@ import ( "time" "github.com/MakeNowJust/heredoc" - "github.com/odpf/optimus/config" cli "github.com/spf13/cobra" + + "github.com/odpf/optimus/config" ) const ( diff --git a/cmd/commands.go b/cmd/commands.go index d124193c63..a2e4606679 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -13,14 +13,15 @@ import ( grpc_retry "github.com/grpc-ecosystem/go-grpc-middleware/retry" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/mattn/go-isatty" - "github.com/odpf/optimus/config" - "github.com/odpf/optimus/models" "github.com/odpf/salt/cmdx" "github.com/odpf/salt/term" "github.com/spf13/afero" cli "github.com/spf13/cobra" "go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" "google.golang.org/grpc" + + "github.com/odpf/optimus/config" + "github.com/odpf/optimus/models" ) var ( diff --git a/cmd/config.go b/cmd/config.go index 1385d48128..cfacb74498 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -5,9 +5,10 @@ import ( "io/ioutil" "github.com/AlecAivazis/survey/v2" - "github.com/odpf/optimus/config" cli "github.com/spf13/cobra" "gopkg.in/yaml.v2" + + "github.com/odpf/optimus/config" ) const ( diff --git a/cmd/extension.go b/cmd/extension.go index fd73844a5a..e97b3b80fe 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -8,9 +8,10 @@ import ( "strings" "github.com/google/go-github/github" + cli "github.com/spf13/cobra" + "github.com/odpf/optimus/config" "github.com/odpf/optimus/extension" - cli "github.com/spf13/cobra" ) func addExtensionCommand(cmd *cli.Command) { diff --git a/cmd/job.go b/cmd/job.go index a7a89aeefb..74d3f141d9 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -1,8 +1,9 @@ package cmd import ( - "github.com/odpf/optimus/config" cli "github.com/spf13/cobra" + + "github.com/odpf/optimus/config" ) func jobCommand() *cli.Command { diff --git a/cmd/logger.go b/cmd/logger.go index 2dab93b3c3..bebacf7bc7 100644 --- a/cmd/logger.go +++ b/cmd/logger.go @@ -4,9 +4,10 @@ import ( "fmt" "os" - "github.com/odpf/optimus/config" "github.com/odpf/salt/log" "github.com/sirupsen/logrus" + + "github.com/odpf/optimus/config" ) type loggerType int diff --git a/cmd/replay.go b/cmd/replay.go index 961aa39639..41e482394a 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -4,10 +4,11 @@ import ( "strings" "time" + cli "github.com/spf13/cobra" + pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" "github.com/odpf/optimus/config" "github.com/odpf/optimus/core/set" - cli "github.com/spf13/cobra" ) const ( diff --git a/cmd/resource.go b/cmd/resource.go index 397e61cfaa..bcc9ab13c5 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -9,13 +9,14 @@ import ( "strings" "github.com/AlecAivazis/survey/v2" + "github.com/odpf/salt/log" + cli "github.com/spf13/cobra" + "github.com/odpf/optimus/config" "github.com/odpf/optimus/models" "github.com/odpf/optimus/store" "github.com/odpf/optimus/store/local" "github.com/odpf/optimus/utils" - "github.com/odpf/salt/log" - cli "github.com/spf13/cobra" ) var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-zA-Z0-9_\-\.]+$`, diff --git a/cmd/serve.go b/cmd/serve.go index 65cb8d19c4..9e952d1714 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -8,10 +8,11 @@ import ( "github.com/hashicorp/go-hclog" hPlugin "github.com/hashicorp/go-plugin" + cli "github.com/spf13/cobra" + "github.com/odpf/optimus/cmd/server" "github.com/odpf/optimus/config" "github.com/odpf/optimus/plugin" - cli "github.com/spf13/cobra" ) func serveCommand() *cli.Command { @@ -48,7 +49,7 @@ func serveCommand() *cli.Command { Level: pluginLogLevel, })); err != nil { hPlugin.CleanupClients() - fmt.Printf("ERROR: %s\n", err.Error()) + l.Error(fmt.Sprintf("ERROR: %s\n", err.Error())) os.Exit(1) } // Make sure we clean up any managed plugins at the end of this @@ -57,7 +58,7 @@ func serveCommand() *cli.Command { // init telemetry teleShutdown, err := config.InitTelemetry(l, conf.Telemetry) if err != nil { - fmt.Printf("ERROR: %s\n", err.Error()) + l.Error(fmt.Sprintf("ERROR: %s\n", err.Error())) os.Exit(1) } defer teleShutdown() diff --git a/cmd/version.go b/cmd/version.go index adfa820c37..a277c0a9ab 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -5,12 +5,13 @@ import ( "fmt" "time" - pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" - "github.com/odpf/optimus/config" - "github.com/odpf/optimus/models" "github.com/odpf/salt/version" cli "github.com/spf13/cobra" "google.golang.org/grpc" + + pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" + "github.com/odpf/optimus/config" + "github.com/odpf/optimus/models" ) const ( diff --git a/config/loader.go b/config/loader.go index e7326db31f..06ce6205b5 100644 --- a/config/loader.go +++ b/config/loader.go @@ -7,7 +7,6 @@ import ( "github.com/odpf/salt/config" "github.com/spf13/afero" - "github.com/spf13/viper" ) @@ -24,7 +23,6 @@ var ( FS = afero.NewReadOnlyFs(afero.NewOsFs()) currPath string execPath string - homePath string ) //nolint:gochecknoinits @@ -40,12 +38,6 @@ func init() { panic(err) } execPath = p - - p, err = os.UserHomeDir() - if err != nil { - panic(err) - } - homePath = p } // LoadClientConfig load the project specific config from these locations: diff --git a/config/loader_test.go b/config/loader_test.go index 5fd840c25e..fd5dafdc86 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -8,9 +8,10 @@ import ( "testing" "time" - "github.com/odpf/optimus/config" "github.com/spf13/afero" "github.com/stretchr/testify/suite" + + "github.com/odpf/optimus/config" ) const clientConfig = ` diff --git a/config/validation_test.go b/config/validation_test.go index 983202434a..4c3ef61adf 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -3,8 +3,9 @@ package config_test import ( "testing" - "github.com/odpf/optimus/config" "github.com/stretchr/testify/suite" + + "github.com/odpf/optimus/config" ) type ValidationTestSuite struct { diff --git a/main.go b/main.go index 4bb0487758..b0a90196ab 100644 --- a/main.go +++ b/main.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "math/rand" - _ "net/http/pprof" "os" "time" From 29a930ea6d97baced66df41166407bee94464268 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Wed, 30 Mar 2022 12:01:09 +0700 Subject: [PATCH 40/54] add: gitignore for new configs --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e5507e6def..d88922ee25 100644 --- a/.gitignore +++ b/.gitignore @@ -19,7 +19,8 @@ jobs /development.env /resources/resource_fs.go __pycache__ -.optimus.yaml +optimus.yaml +config.yaml coverage.txt /api/proto/odpf/**/* From 1ef87aa4cffbd720010d51a0f79992efe51d443b Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 31 Mar 2022 15:20:31 +0700 Subject: [PATCH 41/54] refactor: move setups in server package --- cmd/serve.go | 34 +----------------------------- cmd/server/optimus.go | 48 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 35 deletions(-) diff --git a/cmd/serve.go b/cmd/serve.go index 9e952d1714..132b47bceb 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -6,13 +6,10 @@ import ( "os/signal" "syscall" - "github.com/hashicorp/go-hclog" - hPlugin "github.com/hashicorp/go-plugin" cli "github.com/spf13/cobra" "github.com/odpf/optimus/cmd/server" "github.com/odpf/optimus/config" - "github.com/odpf/optimus/plugin" ) func serveCommand() *cli.Command { @@ -35,35 +32,7 @@ func serveCommand() *cli.Command { return err } - // initiate jsonLogger - l := initLogger(jsonLoggerType, conf.Log) - pluginLogLevel := hclog.Info - if conf.Log.Level == config.LogLevelDebug { - pluginLogLevel = hclog.Debug - } - - // discover and load plugins. TODO: refactor this - if err := plugin.Initialize(hclog.New(&hclog.LoggerOptions{ - Name: "optimus", - Output: os.Stdout, - Level: pluginLogLevel, - })); err != nil { - hPlugin.CleanupClients() - l.Error(fmt.Sprintf("ERROR: %s\n", err.Error())) - os.Exit(1) - } - // Make sure we clean up any managed plugins at the end of this - defer hPlugin.CleanupClients() - - // init telemetry - teleShutdown, err := config.InitTelemetry(l, conf.Telemetry) - if err != nil { - l.Error(fmt.Sprintf("ERROR: %s\n", err.Error())) - os.Exit(1) - } - defer teleShutdown() - l.Info(coloredSuccess("Starting Optimus"), "version", config.BuildVersion) - optimusServer, err := server.New(l, *conf) + optimusServer, err := server.New(*conf) defer optimusServer.Shutdown() if err != nil { return fmt.Errorf("unable to create server: %w", err) @@ -72,7 +41,6 @@ func serveCommand() *cli.Command { sigc := make(chan os.Signal, 1) signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM) <-sigc - l.Info(coloredNotice("Shutting down server")) return nil } diff --git a/cmd/server/optimus.go b/cmd/server/optimus.go index eb11c165c8..9d1af31b7c 100644 --- a/cmd/server/optimus.go +++ b/cmd/server/optimus.go @@ -4,15 +4,18 @@ import ( "context" "fmt" "net/http" + "os" "time" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/hashicorp/go-hclog" "github.com/odpf/salt/log" "github.com/prometheus/client_golang/prometheus" slackapi "github.com/slack-go/slack" "google.golang.org/grpc" "gorm.io/gorm" + hPlugin "github.com/hashicorp/go-plugin" v1handler "github.com/odpf/optimus/api/handler/v1beta1" pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" jobRunCompiler "github.com/odpf/optimus/compiler" @@ -21,6 +24,7 @@ import ( "github.com/odpf/optimus/ext/notify/slack" "github.com/odpf/optimus/job" "github.com/odpf/optimus/models" + "github.com/odpf/optimus/plugin" "github.com/odpf/optimus/service" "github.com/odpf/optimus/store/postgres" "github.com/odpf/optimus/utils" @@ -42,11 +46,10 @@ type OptimusServer struct { cleanupFn []func() } -func New(l log.Logger, conf config.ServerConfig) (*OptimusServer, error) { +func New(conf config.ServerConfig) (*OptimusServer, error) { addr := fmt.Sprintf("%s:%d", conf.Serve.Host, conf.Serve.Port) server := &OptimusServer{ conf: conf, - logger: l, serverAddr: addr, } @@ -55,6 +58,9 @@ func New(l log.Logger, conf config.ServerConfig) (*OptimusServer, error) { } setupFns := []setupFn{ + server.setupLogger, + server.setupPlugins, + server.setupTelemetry, server.setupAppKey, server.setupDB, server.setupGRPCServer, @@ -69,11 +75,48 @@ func New(l log.Logger, conf config.ServerConfig) (*OptimusServer, error) { } } + server.logger.Info("Starting Optimus", "version", config.BuildVersion) server.startListening() return server, nil } +func (s *OptimusServer) setupLogger() error { + s.logger = log.NewLogrus( + log.LogrusWithLevel(s.conf.Log.Level.String()), + log.LogrusWithWriter(os.Stderr), + ) + return nil +} + +func (s *OptimusServer) setupPlugins() error { + pluginLogLevel := hclog.Info + if s.conf.Log.Level == config.LogLevelDebug { + pluginLogLevel = hclog.Debug + } + + pluginLoggerOpt := &hclog.LoggerOptions{ + Name: "optimus", + Output: os.Stdout, + Level: pluginLogLevel, + } + pluginLogger := hclog.New(pluginLoggerOpt) + s.cleanupFn = append(s.cleanupFn, hPlugin.CleanupClients) + + // discover and load plugins. + return plugin.Initialize(pluginLogger) +} + +func (s *OptimusServer) setupTelemetry() error { + teleShutdown, err := config.InitTelemetry(s.logger, s.conf.Telemetry) + if err != nil { + return err + } + + s.cleanupFn = append(s.cleanupFn, teleShutdown) + return nil +} + func (s *OptimusServer) setupAppKey() error { var err error s.appKey, err = models.NewApplicationSecret(s.conf.Serve.AppKey) @@ -130,6 +173,7 @@ func (s *OptimusServer) startListening() { } func (s *OptimusServer) Shutdown() { + s.logger.Info("Shutting down server") if s.httpServer != nil { // Create a deadline to wait for server ctxProxy, cancelProxy := context.WithTimeout(context.Background(), shutdownWait) From d1963ef7b1a409781be860ff61dcdba81c33fc77 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 31 Mar 2022 15:37:09 +0700 Subject: [PATCH 42/54] refactor: logger & hplugin imported --- cmd/admin.go | 4 ++-- cmd/backup.go | 4 ++-- cmd/config.go | 2 +- cmd/deploy.go | 2 +- cmd/extension.go | 3 +-- cmd/job.go | 4 ++-- cmd/logger.go | 37 +++++++++++++------------------------ cmd/replay.go | 4 ++-- cmd/resource.go | 4 ++-- cmd/secret.go | 4 ++-- cmd/server/optimus.go | 2 +- cmd/version.go | 4 ++-- 12 files changed, 31 insertions(+), 43 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index eff220e96e..3b6bb50f1f 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -11,7 +11,7 @@ import ( func adminCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} - l := initLogger(plainLoggerType, conf.Log) + l := initDefaultLogger() cmd := &cli.Command{ Use: "admin", @@ -26,7 +26,7 @@ func adminCommand() *cli.Command { if err != nil { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) return nil } diff --git a/cmd/backup.go b/cmd/backup.go index faa8c324ad..11fdbf0934 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -16,7 +16,7 @@ const ( func backupCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} - l := initLogger(plainLoggerType, conf.Log) + l := initDefaultLogger() cmd := &cli.Command{ Use: "backup", @@ -37,7 +37,7 @@ func backupCommand() *cli.Command { if err != nil { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) return nil } diff --git a/cmd/config.go b/cmd/config.go index cfacb74498..3c96aad6e0 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -98,7 +98,7 @@ func configInitCommand() *cli.Command { return err } - l := initLogger(plainLoggerType, conf.Log) + l := initClientLogger(conf.Log) l.Info(coloredSuccess("Configuration initialised successfully")) return nil }, diff --git a/cmd/deploy.go b/cmd/deploy.go index 6d8a8f86ff..c6750ad2ba 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -62,7 +62,7 @@ func deployCommand() *cli.Command { return err } - l := initLogger(plainLoggerType, conf.Log) + l := initClientLogger(conf.Log) datastoreSpecFs := getDatastoreSpecFs(conf.Namespaces) l.Info(fmt.Sprintf("Deploying project: %s to %s", conf.Project.Name, conf.Host)) diff --git a/cmd/extension.go b/cmd/extension.go index e97b3b80fe..5132daf642 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -10,7 +10,6 @@ import ( "github.com/google/go-github/github" cli "github.com/spf13/cobra" - "github.com/odpf/optimus/config" "github.com/odpf/optimus/extension" ) @@ -55,7 +54,7 @@ func extensionCommand(ctx context.Context, extension *extension.Extension) *cli. func extensionInstallCommand(ctx context.Context, installer extension.Installer) *cli.Command { var ( alias string - l = initLogger(plainLoggerType, config.LogConfig{}) + l = initDefaultLogger() ) installCmd := &cli.Command{ diff --git a/cmd/job.go b/cmd/job.go index 74d3f141d9..1506bd9358 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -9,7 +9,7 @@ import ( func jobCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} - l := initLogger(plainLoggerType, conf.Log) + l := initDefaultLogger() cmd := &cli.Command{ Use: "job", @@ -27,7 +27,7 @@ func jobCommand() *cli.Command { if err != nil { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) return nil } diff --git a/cmd/logger.go b/cmd/logger.go index bebacf7bc7..032f23c977 100644 --- a/cmd/logger.go +++ b/cmd/logger.go @@ -2,7 +2,6 @@ package cmd import ( "fmt" - "os" "github.com/odpf/salt/log" "github.com/sirupsen/logrus" @@ -10,13 +9,6 @@ import ( "github.com/odpf/optimus/config" ) -type loggerType int - -const ( - jsonLoggerType loggerType = iota - plainLoggerType -) - type plainFormatter int func (p *plainFormatter) Format(entry *logrus.Entry) ([]byte, error) { @@ -30,23 +22,20 @@ func (p *plainFormatter) Format(entry *logrus.Entry) ([]byte, error) { return []byte(fmt.Sprintf("%s\n", entry.Message)), nil } -func initLogger(t loggerType, conf config.LogConfig) log.Logger { - if conf.Level == "" { - conf.Level = config.LogLevelInfo - } +func initDefaultLogger() log.Logger { + return log.NewLogrus( + log.LogrusWithLevel(config.LogLevelInfo.String()), + log.LogrusWithFormatter(new(plainFormatter)), + ) +} - switch t { - case jsonLoggerType: - return log.NewLogrus( - log.LogrusWithLevel(conf.Level.String()), - log.LogrusWithWriter(os.Stderr), - ) - case plainLoggerType: - return log.NewLogrus( - log.LogrusWithLevel(conf.Level.String()), - log.LogrusWithFormatter(new(plainFormatter)), - ) +func initClientLogger(conf config.LogConfig) log.Logger { + if conf.Level == "" { + return initDefaultLogger() } - return nil + return log.NewLogrus( + log.LogrusWithLevel(conf.Level.String()), + log.LogrusWithFormatter(new(plainFormatter)), + ) } diff --git a/cmd/replay.go b/cmd/replay.go index 41e482394a..76194209cb 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -54,7 +54,7 @@ func formatRunsPerJobInstance(instance *pb.ReplayExecutionTreeNode, taskReruns m func replayCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} - l := initLogger(plainLoggerType, conf.Log) + l := initDefaultLogger() cmd := &cli.Command{ Use: "replay", @@ -72,7 +72,7 @@ func replayCommand() *cli.Command { if err != nil { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) return nil } diff --git a/cmd/resource.go b/cmd/resource.go index bcc9ab13c5..9d43ad55ba 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -25,7 +25,7 @@ var validateResourceName = utils.ValidatorFactory.NewFromRegex(`^[a-zA-Z0-9][a-z func resourceCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} - l := initLogger(plainLoggerType, conf.Log) + l := initDefaultLogger() cmd := &cli.Command{ Use: "resource", @@ -42,7 +42,7 @@ func resourceCommand() *cli.Command { if err != nil { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) return nil } diff --git a/cmd/secret.go b/cmd/secret.go index 4dbf4d9cf4..da96ee132e 100644 --- a/cmd/secret.go +++ b/cmd/secret.go @@ -29,7 +29,7 @@ const ( func secretCommand() *cli.Command { var configFilePath string conf := &config.ClientConfig{} - l := initLogger(plainLoggerType, conf.Log) + l := initDefaultLogger() cmd := &cli.Command{ Use: "secret", @@ -43,7 +43,7 @@ func secretCommand() *cli.Command { if err != nil { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) return nil } diff --git a/cmd/server/optimus.go b/cmd/server/optimus.go index 9d1af31b7c..7b843bb866 100644 --- a/cmd/server/optimus.go +++ b/cmd/server/optimus.go @@ -9,13 +9,13 @@ import ( grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/hashicorp/go-hclog" + hPlugin "github.com/hashicorp/go-plugin" "github.com/odpf/salt/log" "github.com/prometheus/client_golang/prometheus" slackapi "github.com/slack-go/slack" "google.golang.org/grpc" "gorm.io/gorm" - hPlugin "github.com/hashicorp/go-plugin" v1handler "github.com/odpf/optimus/api/handler/v1beta1" pb "github.com/odpf/optimus/api/proto/odpf/optimus/core/v1beta1" jobRunCompiler "github.com/odpf/optimus/compiler" diff --git a/cmd/version.go b/cmd/version.go index a277c0a9ab..d3e6079027 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -34,7 +34,7 @@ func versionCommand() *cli.Command { c.RunE = func(c *cli.Command, args []string) error { // initiate logger - l := initLogger(plainLoggerType, config.LogConfig{}) + l := initDefaultLogger() // Print client version l.Info(fmt.Sprintf("Client: %s-%s", coloredNotice(config.BuildVersion), coloredNotice(config.BuildCommit))) @@ -51,7 +51,7 @@ func versionCommand() *cli.Command { return err } - l = initLogger(plainLoggerType, conf.Log) + l = initClientLogger(conf.Log) srvVer, err := getVersionRequest(config.BuildVersion, conf.Host) if err != nil { From ae75900c5de058f471c4fcde7a18bdac7f89f500 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Thu, 31 Mar 2022 15:42:30 +0700 Subject: [PATCH 43/54] fix: type load config server --- cmd/serve.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/serve.go b/cmd/serve.go index 132b47bceb..436035ad34 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -23,7 +23,7 @@ func serveCommand() *cli.Command { }, } - cmd.Flags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.Flags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for server configuration") cmd.RunE = func(c *cli.Command, args []string) error { // TODO: find a way to load the config in one place From aadbf85418d124176b0e50853caec89960e60ad6 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 10:51:37 +0700 Subject: [PATCH 44/54] fix: missing --config flags --- cmd/admin.go | 3 +++ cmd/backup.go | 3 +++ cmd/job.go | 4 ++-- cmd/replay.go | 5 +++-- cmd/resource.go | 3 +++ cmd/secret.go | 3 +++ 6 files changed, 17 insertions(+), 4 deletions(-) diff --git a/cmd/admin.go b/cmd/admin.go index 3b6bb50f1f..8a2499788c 100644 --- a/cmd/admin.go +++ b/cmd/admin.go @@ -18,6 +18,9 @@ func adminCommand() *cli.Command { Short: "Internal administration commands", Hidden: true, } + + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { // TODO: find a way to load the config in one place var err error diff --git a/cmd/backup.go b/cmd/backup.go index 11fdbf0934..85c7c56b57 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -29,6 +29,9 @@ func backupCommand() *cli.Command { "group:core": "true", }, } + + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { // TODO: find a way to load the config in one place var err error diff --git a/cmd/job.go b/cmd/job.go index 1506bd9358..80af5e7034 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -19,6 +19,8 @@ func jobCommand() *cli.Command { }, } + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { // TODO: find a way to load the config in one place var err error @@ -32,8 +34,6 @@ func jobCommand() *cli.Command { return nil } - cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") - cmd.AddCommand(jobCreateCommand(l, conf)) cmd.AddCommand(jobAddHookCommand(l, conf)) cmd.AddCommand(jobRenderTemplateCommand(l, conf)) diff --git a/cmd/replay.go b/cmd/replay.go index 76194209cb..250e914955 100644 --- a/cmd/replay.go +++ b/cmd/replay.go @@ -64,6 +64,9 @@ func replayCommand() *cli.Command { "group:core": "true", }, } + + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { // TODO: find a way to load the config in one place var err error @@ -77,8 +80,6 @@ func replayCommand() *cli.Command { return nil } - cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") - cmd.AddCommand(replayCreateCommand(l, conf)) cmd.AddCommand(replayStatusCommand(l, conf)) cmd.AddCommand(replayListCommand(l, conf)) diff --git a/cmd/resource.go b/cmd/resource.go index 9d43ad55ba..530cbdd88a 100644 --- a/cmd/resource.go +++ b/cmd/resource.go @@ -34,6 +34,9 @@ func resourceCommand() *cli.Command { "group:core": "true", }, } + + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { // TODO: find a way to load the config in one place var err error diff --git a/cmd/secret.go b/cmd/secret.go index da96ee132e..aca2f8ba3c 100644 --- a/cmd/secret.go +++ b/cmd/secret.go @@ -35,6 +35,9 @@ func secretCommand() *cli.Command { Use: "secret", Short: "Manage secrets to be used in jobs", } + + cmd.PersistentFlags().StringVarP(&configFilePath, "config", "c", configFilePath, "File path for client configuration") + cmd.PersistentPreRunE = func(cmd *cli.Command, args []string) error { // TODO: find a way to load the config in one place var err error From 1ed316c529c549094ab080776d5459f84c1dadd5 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 11:26:58 +0700 Subject: [PATCH 45/54] fix: default scheduler in config server --- config/config_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config_server.go b/config/config_server.go index d6354e9b78..8ca737c8ef 100644 --- a/config/config_server.go +++ b/config/config_server.go @@ -34,7 +34,7 @@ type DBConfig struct { } type SchedulerConfig struct { - Name string `mapstructure:"name" default:"airflow2"` + Name string `mapstructure:"name" default:"airflow"` SkipInit bool `mapstructure:"skip_init"` RaftAddr string `mapstructure:"raft_addr"` GossipAddr string `mapstructure:"gossip_addr"` From 46547b0d5336197ae36f06643ea7c0658c100fc5 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 14:05:16 +0700 Subject: [PATCH 46/54] refactor: passing logger in extension command --- cmd/extension.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/extension.go b/cmd/extension.go index 5132daf642..f7489ca2ec 100644 --- a/cmd/extension.go +++ b/cmd/extension.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/google/go-github/github" + "github.com/odpf/salt/log" cli "github.com/spf13/cobra" "github.com/odpf/optimus/extension" @@ -42,20 +43,18 @@ func addExtensionCommand(cmd *cli.Command) { } func extensionCommand(ctx context.Context, extension *extension.Extension) *cli.Command { + l := initDefaultLogger() c := &cli.Command{ Use: "extension SUBCOMMAND", Aliases: []string{"ext"}, Short: "Operate with extension", } - c.AddCommand(extensionInstallCommand(ctx, extension)) + c.AddCommand(extensionInstallCommand(ctx, extension, l)) return c } -func extensionInstallCommand(ctx context.Context, installer extension.Installer) *cli.Command { - var ( - alias string - l = initDefaultLogger() - ) +func extensionInstallCommand(ctx context.Context, installer extension.Installer, l log.Logger) *cli.Command { + var alias string installCmd := &cli.Command{ Use: "install OWNER/REPO", From ef05cecf16501c8ade2dae908fa44d95eac8d0e7 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 15:23:06 +0700 Subject: [PATCH 47/54] fix: initialize pluginRegistry --- cmd/deploy.go | 9 ++++++++- cmd/job.go | 7 +++++++ cmd/plugins.go | 31 +++++++++++++++++++++++++++++++ cmd/version.go | 7 +++++++ 4 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 cmd/plugins.go diff --git a/cmd/deploy.go b/cmd/deploy.go index c6750ad2ba..38df0191c3 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -61,8 +61,15 @@ func deployCommand() *cli.Command { if err != nil { return err } - l := initClientLogger(conf.Log) + + // TODO: refactor initialize client deps + pluginCleanFn, err := initializeClientPlugins(conf.Log.Level) + defer pluginCleanFn() + if err != nil { + return err + } + datastoreSpecFs := getDatastoreSpecFs(conf.Namespaces) l.Info(fmt.Sprintf("Deploying project: %s to %s", conf.Project.Name, conf.Host)) diff --git a/cmd/job.go b/cmd/job.go index 80af5e7034..d91faa98fe 100644 --- a/cmd/job.go +++ b/cmd/job.go @@ -31,6 +31,13 @@ func jobCommand() *cli.Command { } l = initClientLogger(conf.Log) + // TODO: refactor initialize client deps + pluginCleanFn, err := initializeClientPlugins(conf.Log.Level) + defer pluginCleanFn() + if err != nil { + return err + } + return nil } diff --git a/cmd/plugins.go b/cmd/plugins.go new file mode 100644 index 0000000000..899d2feb43 --- /dev/null +++ b/cmd/plugins.go @@ -0,0 +1,31 @@ +package cmd + +import ( + "os" + + "github.com/hashicorp/go-hclog" + hPlugin "github.com/hashicorp/go-plugin" + + "github.com/odpf/optimus/config" + "github.com/odpf/optimus/plugin" +) + +// TODO: need to refactor this along side with refactoring the client commands +// Plugin's helper to initialize plugins on client side +func initializeClientPlugins(logLevel config.LogLevel) (cleanFn func(), err error) { + pluginLogLevel := hclog.Info + if logLevel == config.LogLevelDebug { + pluginLogLevel = hclog.Debug + } + + pluginLoggerOpt := &hclog.LoggerOptions{ + Name: "optimus", + Output: os.Stdout, + Level: pluginLogLevel, + } + pluginLogger := hclog.New(pluginLoggerOpt) + + // discover and load plugins. + err = plugin.Initialize(pluginLogger) + return hPlugin.CleanupClients, err +} diff --git a/cmd/version.go b/cmd/version.go index d3e6079027..7b571f180c 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -36,6 +36,13 @@ func versionCommand() *cli.Command { // initiate logger l := initDefaultLogger() + // TODO: refactor initialize client deps + pluginCleanFn, err := initializeClientPlugins(config.LogLevelInfo) + defer pluginCleanFn() + if err != nil { + return err + } + // Print client version l.Info(fmt.Sprintf("Client: %s-%s", coloredNotice(config.BuildVersion), coloredNotice(config.BuildCommit))) From d849fb3ae8e6ecaa562c3838f784d31d8e8428e2 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 15:25:06 +0700 Subject: [PATCH 48/54] refactor: add TODO init paths config --- config/loader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/loader.go b/config/loader.go index 06ce6205b5..9dcd9d356e 100644 --- a/config/loader.go +++ b/config/loader.go @@ -26,7 +26,7 @@ var ( ) //nolint:gochecknoinits -func init() { +func init() { // TODO: move paths initialization outside init() p, err := os.Getwd() if err != nil { panic(err) From 39b26d2ef82f89fc2fda0d5d34de1e5fbafbab4b Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 15:28:25 +0700 Subject: [PATCH 49/54] refactor: add TODO for validation --- cmd/version.go | 2 +- config/validation.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 7b571f180c..4349bae642 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -54,7 +54,7 @@ func versionCommand() *cli.Command { return err } - if err := config.Validate(conf); err != nil { + if err := config.Validate(conf); err != nil { // experiment for client validation return err } diff --git a/config/validation.go b/config/validation.go index 1da73c02e9..1edef88b29 100644 --- a/config/validation.go +++ b/config/validation.go @@ -10,7 +10,7 @@ import ( ) // Validate validate the config as an input. If not valid, it returns error -func Validate(conf Config) error { +func Validate(conf Config) error { // TODO: call Validate explicitly when it needed switch c := conf.(type) { case *ClientConfig: return validateClientConfig(c) From 545f91f6e64f69bd88f333041d8150741cfdfc80 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 16:21:42 +0700 Subject: [PATCH 50/54] fix: executable dir path --- config/loader.go | 3 ++- config/loader_test.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/config/loader.go b/config/loader.go index 9dcd9d356e..6dd4836bad 100644 --- a/config/loader.go +++ b/config/loader.go @@ -3,6 +3,7 @@ package config import ( "fmt" "os" + "path/filepath" "strings" "github.com/odpf/salt/config" @@ -37,7 +38,7 @@ func init() { // TODO: move paths initialization outside init() if err != nil { panic(err) } - execPath = p + execPath = filepath.Dir(p) } // LoadClientConfig load the project specific config from these locations: diff --git a/config/loader_test.go b/config/loader_test.go index fd5dafdc86..e5a37f11ae 100644 --- a/config/loader_test.go +++ b/config/loader_test.go @@ -4,6 +4,7 @@ import ( "io/fs" "os" "path" + "path/filepath" "strings" "testing" "time" @@ -80,7 +81,7 @@ func (s *ConfigTestSuite) SetupTest() { p, err = os.Executable() s.Require().NoError(err) - s.execPath = p + s.execPath = filepath.Dir(p) s.a.Fs.MkdirAll(s.execPath, fs.ModeTemporary) config.FS = s.a.Fs From e0f4a35712d643439328ce59799561b9e3f4899a Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Fri, 1 Apr 2022 16:23:22 +0700 Subject: [PATCH 51/54] refactor: remove global env as a reasons when server is not reachable --- cmd/commands.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/commands.go b/cmd/commands.go index a2e4606679..71e63f18d1 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -35,8 +35,7 @@ var ( return errors.New(heredoc.Docf(`Unable to reach optimus server at %s, this can happen due to following reasons: 1. Check if you are connected to internet 2. Is the host correctly configured in optimus config - 3. Is OPTIMUS_HOST env incorrectly set - 4. Is Optimus server currently unreachable`, host)) + 3. Is Optimus server currently unreachable`, host)) } ) From bb6f797643fb476e6a98801d1ff4be2bc24c09aa Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 4 Apr 2022 10:53:02 +0700 Subject: [PATCH 52/54] docs: update config docs --- README.md | 3 +- docs/docs/getting-started/configuration.md | 161 +++++++++------------ 2 files changed, 70 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index 3b3e9f2d0d..c12130ea85 100644 --- a/README.md +++ b/README.md @@ -97,8 +97,7 @@ Optimus service can be started with $ ./optimus serve ``` -`serve` command has few required configurations that needs to be set for it to start. Configuration can either be stored -in `.optimus.yaml` file or set as environment variable. Read more about it in [getting started](https://odpf.github.io/optimus/docs/getting-started/configuration). +`serve` command has few required configurations that needs to be set for it to start. Read more about it in [getting started](https://odpf.github.io/optimus/docs/getting-started/configuration). ## Compatibility diff --git a/docs/docs/getting-started/configuration.md b/docs/docs/getting-started/configuration.md index 701f2e5dee..0a084fbe45 100644 --- a/docs/docs/getting-started/configuration.md +++ b/docs/docs/getting-started/configuration.md @@ -3,116 +3,93 @@ id: configuration title: Configurations --- -Optimus can be configured with `.optimus.yaml` file. An example of such is: -```yaml -version: 1 +# Project Configuration (Client) +See client configuration example on `optimus.sample.yaml` -# used to connect optimus service -host: localhost:9100 - -# project specification -project: - - # name of the Optimus project - name: sample_project - - # project level variables usable in specifications - config: {} - -# namespace specification of the jobs and resources -namespace: - - # namespace name - name: sample_namespace - - jobs: - # folder where job specifications are stored - path: "job" - - datastore: - # optimus is capable of supporting multiple datastores - type: bigquery - # path where resource spec for BQ are stored - path: "bq" - # backup configurations of a datastore - backup: - # backup result age until expired - default '720h' - ttl: 168h - # where backup result should be located - default 'optimus_backup' - dataset: archive - # backup result prefix table name - default 'backup' - prefix: archive - - # namespace level variables usable in specifications - config: {} - -# for configuring optimus service locally -serve: - - # port to listen on - port: 9100 - - # host to listen on - host: localhost - - # this gets injected in compiled dags to reach back out to optimus service - # when they run - ingress_host: optimus.example.io:80 - - # 32 char hash used for encrypting secrets - app_key: Yjo4a0jn1NvYdq79SADC/KaVv9Wu0Ffc - - # database configurations - db: - # database connection string - dsn: postgres://user:password@localhost:5432/database?sslmode=disable - - max_idle_connection: 5 - max_open_connection: 10 - -# logging configuration -log: - # debug, info, warning, error, fatal - default 'info' - level: debug +Optimus project configuration (later on client configuration) can be loaded from file (use `--config` flag), or `optimus.yaml` file in current working directory where the optimus command is executed. +--- +**1. Using --config flag** +```sh +$ optimus deploy --config /path/to/config/file.yaml ``` +--- +**2. Using default optimus.yaml file** +```sh +$ tree +. # current project structure +├── namespace-1 +│   └── jobs +│   └── resources +├── namespace-2 +│   └── jobs +│   └── resources +└── optimus.yaml # use this file +$ optimus deploy +``` + +--- + +If both are exist, then use the file config defined in `--config` flag. + +This configuration file should not be checked in version control. +# Server Configuration +See server configuration example on `config.sample.yaml` + +Optimus server configuration can be loaded from file (use `--config` flag), environment variable `OPTIMUS_`, or `config.yaml` file in executable directory. + +--- +**1. Using --config flag** +```sh +$ optimus serve --config /path/to/config/file.yaml +``` + +--- +**2. Using environment variable** -This configuration file should not be checked in version control. All the configs can also be passed as environment -variables using `OPTIMUS_` convention, for example to set client host `OPTIMUS_HOST=localhost:9100` to set -database connection string `OPTIMUS_SERVE_DB_DSN=postgres://dbconenctionurl`. +All the configs can be passed as environment variables using `OPTIMUS_` convention. `` is the key name of config using `_` as the path delimiter to concatenate between keys. -Assuming the following configuration layout: +For example, to use environment variable, assuming the following configuration layout: ```yaml -host: localhost:9100 +version: 1 serve: port: 9100 app_key: randomhash ``` -Key `host` can be set as an environment variable by upper-casing its path, using `_` as the -path delimiter and prefixing with `OPTIMUS_`: - -`serve.port` -> `OPTIMUS_SERVE_PORT=9100` -or: -`host` -> `OPTIMUS_HOST=localhost:9100` - -Environment variables always override values from the configuration file. Here are some more examples: +Here is the corresponding environment variable for the above Configuration key | Environment variable | ------------------|----------------------| -host | OPTIMUS_HOST | +version | OPTIMUS_VERSION | +serve.port | OPTIMUS_PORT | serve.app_key | OPTIMUS_SERVE_APP_KEY| +Set the env variable using export +```sh +$ export OPTIMUS_PORT=9100 +``` + +--- +**3. Using default config.yaml from executable binary directory** +```sh +$ which optimus +/usr/local/bin/optimus +``` + +So the `config.yaml` file can be loaded on `/usr/local/bin/config.yaml` + +--- + +If user specify configuration file using `--config` flag, then any configs defined in env variable and default config.yaml from exec directory will not be loaded. + +If user specify the env variable and user also have config.yaml from exec directory, then any key configs from env variable will override the key configs defined in config.yaml from exec directory. + +--- + App key is used to encrypt credentials and can be randomly generated using ```shell head -c 50 /dev/random | base64 ``` -Just take the first 32 characters of the string. - -Configuration file can be stored in following locations: -```shell -./ -/ -~/.optimus/ -``` \ No newline at end of file +Just take the first 32 characters of the string. \ No newline at end of file From 06df79e9a14fe63989f54bb18ee1a8617b0a7f59 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 4 Apr 2022 13:18:16 +0700 Subject: [PATCH 53/54] refactor: initialize dsRepo on backup create command --- cmd/backup_create.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/backup_create.go b/cmd/backup_create.go index 242ffb9965..994b373785 100644 --- a/cmd/backup_create.go +++ b/cmd/backup_create.go @@ -43,12 +43,11 @@ func backupCreateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { backupCmd.RunE = func(cmd *cli.Command, args []string) error { var err error - dsRepo := models.DatastoreRegistry namespace, err := askToSelectNamespace(l, conf) if err != nil { return err } - if storerName, err = extractDatastoreName(dsRepo, storerName); err != nil { + if storerName, err = extractDatastoreName(storerName); err != nil { return err } if resourceName, err = extractResourceName(resourceName); err != nil { @@ -117,7 +116,8 @@ func backupCreateCommand(l log.Logger, conf *config.ClientConfig) *cli.Command { return backupCmd } -func extractDatastoreName(dsRepo models.DatastoreRepo, storerName string) (string, error) { +func extractDatastoreName(storerName string) (string, error) { + dsRepo := models.DatastoreRegistry availableStorer := []string{} for _, s := range dsRepo.GetAll() { availableStorer = append(availableStorer, s.Name()) From 14df5056fa393953ffe1a5e2cd289aba1a6b1629 Mon Sep 17 00:00:00 2001 From: Dery Rahman Ahaddienata Date: Mon, 4 Apr 2022 14:34:26 +0700 Subject: [PATCH 54/54] refactor: remove Validate func --- cmd/version.go | 2 +- config/validation.go | 15 ++------------ config/validation_test.go | 41 ++++++++++++++++----------------------- 3 files changed, 20 insertions(+), 38 deletions(-) diff --git a/cmd/version.go b/cmd/version.go index 4349bae642..f084c24ef2 100644 --- a/cmd/version.go +++ b/cmd/version.go @@ -54,7 +54,7 @@ func versionCommand() *cli.Command { return err } - if err := config.Validate(conf); err != nil { // experiment for client validation + if err := config.ValidateClientConfig(conf); err != nil { // experiment for client validation return err } diff --git a/config/validation.go b/config/validation.go index 1edef88b29..77ea2ae0a7 100644 --- a/config/validation.go +++ b/config/validation.go @@ -9,18 +9,7 @@ import ( validation "github.com/go-ozzo/ozzo-validation/v4" ) -// Validate validate the config as an input. If not valid, it returns error -func Validate(conf Config) error { // TODO: call Validate explicitly when it needed - switch c := conf.(type) { - case *ClientConfig: - return validateClientConfig(c) - case *ServerConfig: - return validateServerConfig(c) - } - return errors.New("config type is not valid, use ClientConfig or ServerConfig instead") -} - -func validateClientConfig(conf *ClientConfig) error { +func ValidateClientConfig(conf *ClientConfig) error { // implement this return validation.ValidateStruct(conf, validation.Field(&conf.Version, validation.Required), @@ -39,7 +28,7 @@ func validateClientConfig(conf *ClientConfig) error { ) } -func validateServerConfig(conf *ServerConfig) error { +func ValidateServerConfig(conf *ServerConfig) error { // implement this return nil } diff --git a/config/validation_test.go b/config/validation_test.go index 4c3ef61adf..f7fadc0704 100644 --- a/config/validation_test.go +++ b/config/validation_test.go @@ -21,37 +21,30 @@ func TestValidation(t *testing.T) { suite.Run(t, new(ValidationTestSuite)) } -func (s *ValidationTestSuite) TestValidate() { - s.Run("WhenValidateClientConfig", func() { - s.Run("WhenConfigIsValid", func() { - err := config.Validate(s.defaultClientConfig) - s.Assert().NoError(err) - }) - - s.Run("WhenNamespacesIsDuplicated", func() { - clientConfig := s.defaultClientConfig - namespaces := clientConfig.Namespaces - namespaces = append(namespaces, &config.Namespace{Name: "ns-dup"}) - namespaces = append(namespaces, &config.Namespace{Name: "ns-dup"}) - clientConfig.Namespaces = namespaces - - err := config.Validate(clientConfig) - - s.Assert().Error(err) - }) +func (s *ValidationTestSuite) TestValidateClientConfig() { + s.Run("WhenConfigIsValid", func() { + err := config.ValidateClientConfig(s.defaultClientConfig) + s.Assert().NoError(err) }) - s.Run("WhenValidateServerConfig", func() { - // TODO: implement this - s.T().Skip() - }) + s.Run("WhenNamespacesIsDuplicated", func() { + clientConfig := s.defaultClientConfig + namespaces := clientConfig.Namespaces + namespaces = append(namespaces, &config.Namespace{Name: "ns-dup"}) + namespaces = append(namespaces, &config.Namespace{Name: "ns-dup"}) + clientConfig.Namespaces = namespaces + + err := config.ValidateClientConfig(clientConfig) - s.Run("WhenValidateTypeIsInvalid", func() { - err := config.Validate("invalid-type") s.Assert().Error(err) }) } +func (s *ValidationTestSuite) TestValidateServerConfig() { + // TODO: implement this + s.T().Skip() +} + func (s *ValidationTestSuite) initDefaultClientConfig() { s.defaultClientConfig = &config.ClientConfig{} s.defaultClientConfig.Version = config.Version(1)