From 3c65cbcca861536c32ba7e1f8fc9eee47c6140e2 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 30 Mar 2022 11:25:18 +0530 Subject: [PATCH 01/20] feat: provider framework to pass creds to k8s/helm provider --- app/app.go | 45 ++++++++++-- domain/provider.go | 23 ++++++ go.mod | 2 +- modules/firehose/module.go | 100 ++++++++++++++++++++++++++- pkg/provider/helm/provider.go | 20 ++++-- pkg/provider/helm/release.go | 46 ++++++------ store/mongodb/provider_repository.go | 53 ++++++++++++++ store/store.go | 17 +++-- 8 files changed, 266 insertions(+), 40 deletions(-) create mode 100644 domain/provider.go create mode 100644 store/mongodb/provider_repository.go diff --git a/app/app.go b/app/app.go index 6204a3fc..5ed15bac 100644 --- a/app/app.go +++ b/app/app.go @@ -3,16 +3,18 @@ package app import ( "context" "fmt" + "net/http" + "time" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" "github.com/odpf/entropy/modules/firehose" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/pkg/module" + "github.com/odpf/entropy/pkg/provider/helm" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" "github.com/odpf/entropy/store/inmemory" "github.com/odpf/entropy/store/mongodb" - "net/http" - "time" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" @@ -33,6 +35,8 @@ import ( "google.golang.org/grpc" ) +var providerURN = "provider_name-kubernetes" + // Config contains the application configuration type Config struct { Service ServiceConfig `mapstructure:"service"` @@ -72,14 +76,31 @@ func RunServer(c *Config) error { mongoStore.Collection(store.ResourceRepositoryName), ) - moduleRepository := inmemory.NewModuleRepository() + providerRepository := mongodb.NewProviderRepository( + mongoStore.Collection(store.ProviderRepositoryName), + ) + moduleRepository := inmemory.NewModuleRepository() err = moduleRepository.Register(log.New(loggerInstance)) if err != nil { return err } - err = moduleRepository.Register(firehose.New()) + providerFetched, err := providerRepository.GetByURN(providerURN) + if err != nil { + return err + } + providerConfig := providerFetched.Configs + + kubeConfig := helm.ToKubeConfig(providerConfig) + + helmConfig := &helm.ProviderConfig{ + Kubernetes: kubeConfig, + } + + hp := helm.NewProvider(helmConfig) + + err = moduleRepository.Register(firehose.New(hp)) if err != nil { return err } @@ -160,5 +181,19 @@ func RunMigrations(c *Config) error { mongoStore.Collection(store.ResourceRepositoryName), ) - return resourceRepository.Migrate() + providerRepository := mongodb.NewProviderRepository( + mongoStore.Collection(store.ProviderRepositoryName), + ) + + err = resourceRepository.Migrate() + if err != nil { + return err + } + + err = providerRepository.Migrate() + if err != nil { + return err + } + + return nil } diff --git a/domain/provider.go b/domain/provider.go new file mode 100644 index 00000000..3febab12 --- /dev/null +++ b/domain/provider.go @@ -0,0 +1,23 @@ +package domain + +import ( + "strings" + "time" +) + +type Provider struct { + Urn string `bson:"urn"` + Name string `bson:"name"` + Kind string `bson:"kind"` + Configs map[string]interface{} `bson:"configs"` + Labels map[string]string `bson:"labels"` + CreatedAt time.Time `bson:"created_at"` + UpdatedAt time.Time `bson:"updated_at"` +} + +func GenerateProviderUrn(res *Provider) string { + return strings.Join([]string{ + sanitizeString(res.Name), + sanitizeString(res.Kind), + }, "-") +} diff --git a/go.mod b/go.mod index d549929f..db23b559 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.17 require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/mcuadros/go-defaults v1.2.0 + github.com/mitchellh/mapstructure v1.4.3 github.com/newrelic/go-agent/v3 v3.12.0 github.com/newrelic/go-agent/v3/integrations/nrgrpc v1.3.1 github.com/odpf/salt v0.0.0-20210929215807-5e1f68b4ec91 @@ -93,7 +94,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 893f44f0..ca91147b 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -3,12 +3,32 @@ package firehose import ( "errors" "fmt" + "reflect" "strings" + "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" + "github.com/odpf/entropy/pkg/provider/helm" gjs "github.com/xeipuuv/gojsonschema" ) +const ( + nameConfigString = "name" + repositoryConfigString = "repository" + chartConfigString = "chart" + versionConfigString = "version" + valuesConfigString = "values" + namespaceConfigString = "namespace" + timeoutConfigString = "timeout" + forceUpdateConfigString = "force_update" + recreatePodsConfigString = "recreate_pods" + waitConfigString = "wait" + waitForJobsConfigString = "wait_for_jobs" + replaceConfigString = "replace" + descriptionConfigString = "description" + CreateNamespaceConfigString = "create_namespace" +) + const configSchemaString = ` { "$schema": "http://json-schema.org/draft-07/schema#", @@ -229,13 +249,14 @@ const configSchemaString = ` type Module struct { schema *gjs.Schema + helm *helm.Provider } func (m *Module) ID() string { return "firehose" } -func New() *Module { +func New(helm *helm.Provider) *Module { schemaLoader := gjs.NewStringLoader(configSchemaString) schema, err := gjs.NewSchema(schemaLoader) if err != nil { @@ -243,15 +264,90 @@ func New() *Module { } return &Module{ schema: schema, + helm: helm, } } func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { + v := reflect.ValueOf(r.Configs[valuesConfigString]) + var values = make(map[string]interface{}) + if v.Kind() == reflect.Map { + for _, key := range v.MapKeys() { + strct := v.MapIndex(key) + values[key.String()] = strct.Interface() + } + } + + var rc helm.ReleaseConfig + if err := mapstructure.Decode(r.Configs, &rc); err != nil { + return domain.ResourceStatusError, err + } + + releaseConfig := helm.DefaultReleaseConfig() + releaseConfig.Values = values + + if r.Configs[nameConfigString] != nil { + releaseConfig.Name = r.Configs[nameConfigString].(string) + } + + if r.Configs[repositoryConfigString] != nil { + releaseConfig.Repository = r.Configs[repositoryConfigString].(string) + } + + if r.Configs[chartConfigString] != nil { + releaseConfig.Chart = r.Configs[chartConfigString].(string) + } + + if r.Configs[versionConfigString] != nil { + releaseConfig.Version = r.Configs[versionConfigString].(string) + } + + if r.Configs[namespaceConfigString] != nil { + releaseConfig.Namespace = r.Configs[namespaceConfigString].(string) + } + + if r.Configs[timeoutConfigString] != nil { + releaseConfig.Timeout = r.Configs[timeoutConfigString].(int) + } + + if r.Configs[descriptionConfigString] != nil { + releaseConfig.Description = r.Configs[descriptionConfigString].(string) + } + + if r.Configs[forceUpdateConfigString] != nil { + releaseConfig.ForceUpdate = r.Configs[forceUpdateConfigString].(bool) + } + + if r.Configs[recreatePodsConfigString] != nil { + releaseConfig.RecreatePods = r.Configs[recreatePodsConfigString].(bool) + } + + if r.Configs[waitConfigString] != nil { + releaseConfig.Wait = r.Configs[waitConfigString].(bool) + } + + if r.Configs[waitForJobsConfigString] != nil { + releaseConfig.WaitForJobs = r.Configs[waitForJobsConfigString].(bool) + } + + if r.Configs[replaceConfigString] != nil { + releaseConfig.Replace = r.Configs[replaceConfigString].(bool) + } + + if r.Configs[CreateNamespaceConfigString] != nil { + releaseConfig.CreateNamespace = r.Configs[CreateNamespaceConfigString].(bool) + } + + _, err := m.helm.Release(releaseConfig) + if err != nil { + return domain.ResourceStatusError, nil + } + return domain.ResourceStatusCompleted, nil } func (m *Module) Validate(r *domain.Resource) error { - resourceLoader := gjs.NewGoLoader(r.Configs) + resourceLoader := gjs.NewGoLoader(r.Configs["values"]) result, err := m.schema.Validate(resourceLoader) if err != nil { return fmt.Errorf("%w: %s", domain.ModuleConfigParseFailed, err) diff --git a/pkg/provider/helm/provider.go b/pkg/provider/helm/provider.go index a3ea42e0..43fb48bd 100644 --- a/pkg/provider/helm/provider.go +++ b/pkg/provider/helm/provider.go @@ -10,15 +10,15 @@ import ( "k8s.io/client-go/tools/clientcmd/api" ) -type providerConfig struct { +type ProviderConfig struct { // HelmDriver - The backend storage driver. Values are - configmap, secret, memory, sql HelmDriver string `default:"secret"` // Kubernetes configuration. Kubernetes KubernetesConfig } -func DefaultProviderConfig() *providerConfig { - defaultProviderConfig := new(providerConfig) +func DefaultProviderConfig() *ProviderConfig { + defaultProviderConfig := new(ProviderConfig) defaults.SetDefaults(defaultProviderConfig) return defaultProviderConfig } @@ -41,11 +41,11 @@ type KubernetesConfig struct { } type Provider struct { - config *providerConfig + config *ProviderConfig cliSettings *cli.EnvSettings } -func NewProvider(config *providerConfig) *Provider { +func NewProvider(config *ProviderConfig) *Provider { return &Provider{config: config, cliSettings: cli.New()} } @@ -80,3 +80,13 @@ func (p *Provider) getActionConfiguration(namespace string) (*action.Configurati } return actionConfig, nil } + +func ToKubeConfig(providerConfig map[string]interface{}) KubernetesConfig { + return KubernetesConfig{ + Host: providerConfig["host"].(string), + Insecure: providerConfig["insecure"].(bool), + ClientCertificate: providerConfig["clientCertificate"].(string), + ClientKey: providerConfig["clientKey"].(string), + ClusterCACertificate: providerConfig["clusterCACertificate"].(string), + } +} diff --git a/pkg/provider/helm/release.go b/pkg/provider/helm/release.go index 371c4db9..7b8fa8c0 100644 --- a/pkg/provider/helm/release.go +++ b/pkg/provider/helm/release.go @@ -18,45 +18,45 @@ import ( var ErrReleaseNotFound = errors.New("release not found") var ErrChartNotApplication = errors.New("helm chart is not an application chart") -type releaseConfig struct { +type ReleaseConfig struct { // Name - Release Name - Name string `valid:"required"` + Name string `json:"name" mapstructure:"name" valid:"required"` // Repository - Repository where to locate the requested chart. If is a URL the chart is installed without installing the repository. - Repository string `valid:"required"` + Repository string `json:"repository" mapstructure:"repository" valid:"required"` // Chart - Chart name to be installed. A path may be used. - Chart string `valid:"required"` + Chart string `json:"chart" mapstructure:"chart" valid:"required"` // Version - Specify the exact chart version to install. If this is not specified, the latest version is installed. - Version string + Version string `json:"version" mapstructure:"version"` // Values - Map of values in to pass to helm. - Values map[string]interface{} + Values map[string]interface{} `json:"values" mapstructure:"values"` // Namespace - Namespace to install the release into. - Namespace string `default:"default"` + Namespace string `json:"namespace" mapstructure:"namespace" default:"default"` // Timeout - Time in seconds to wait for any individual kubernetes operation. - Timeout int `default:"300"` + Timeout int `json:"timeout" mapstructure:"timeout" default:"300"` // ForceUpdate - Force resource update through delete/recreate if needed. - ForceUpdate bool `default:"false"` + ForceUpdate bool `json:"force_update" mapstructure:"force_update" default:"false"` // RecreatePods - Perform pods restart during upgrade/rollback - RecreatePods bool `default:"false"` + RecreatePods bool `json:"recreate_pods" mapstructure:"recreate_pods" default:"false"` // Wait - Will wait until all resources are in a ready state before marking the release as successful. - Wait bool `default:"true"` + Wait bool `json:"wait" mapstructure:"wait" default:"true"` // WaitForJobs - If wait is enabled, will wait until all Jobs have been completed before marking the release as successful. - WaitForJobs bool `default:"false"` + WaitForJobs bool `json:"wait_for_jobs" mapstructure:"wait_for_jobs" default:"false"` // Replace - Re-use the given name, even if that name is already used. This is unsafe in production - Replace bool `default:"false"` + Replace bool `json:"replace" mapstructure:"replace" default:"false"` // Description - Add a custom description - Description string + Description string `json:"description" mapstructure:"description"` // CreateNamespace - Create the namespace if it does not exist - CreateNamespace bool `default:"false"` + CreateNamespace bool `json:"create_namespace" mapstructure:"create_namespace" default:"false"` } -func DefaultReleaseConfig() *releaseConfig { - defaultReleaseConfig := new(releaseConfig) +func DefaultReleaseConfig() *ReleaseConfig { + defaultReleaseConfig := new(ReleaseConfig) defaults.SetDefaults(defaultReleaseConfig) return defaultReleaseConfig } type Release struct { - Config *releaseConfig + Config *ReleaseConfig Output ReleaseOutput } @@ -68,7 +68,7 @@ type ReleaseOutput struct { } // Release - creates or updates a helm release with its configs -func (p *Provider) Release(config *releaseConfig) (*Release, error) { +func (p *Provider) Release(config *ReleaseConfig) (*Release, error) { releaseExists, _ := p.resourceReleaseExists(config.Name, config.Namespace) if releaseExists { return p.update(config) @@ -76,7 +76,7 @@ func (p *Provider) Release(config *releaseConfig) (*Release, error) { return p.create(config) } -func (p *Provider) create(config *releaseConfig) (*Release, error) { +func (p *Provider) create(config *ReleaseConfig) (*Release, error) { actionConfig, err := p.getActionConfiguration(config.Namespace) if err != nil { return nil, fmt.Errorf("error while getting action configuration : %w", err) @@ -155,7 +155,7 @@ func (p *Provider) create(config *releaseConfig) (*Release, error) { }, nil } -func (p *Provider) update(config *releaseConfig) (*Release, error) { +func (p *Provider) update(config *ReleaseConfig) (*Release, error) { var rel *release.Release var err error @@ -230,7 +230,7 @@ func (p *Provider) update(config *releaseConfig) (*Release, error) { }, nil } -func (p *Provider) chartPathOptions(config *releaseConfig) (*action.ChartPathOptions, string) { +func (p *Provider) chartPathOptions(config *ReleaseConfig) (*action.ChartPathOptions, string) { repositoryURL, chartName := resolveChartName(config.Repository, strings.TrimSpace(config.Chart)) version := getVersion(config.Version) @@ -261,7 +261,7 @@ func getVersion(version string) string { return strings.TrimSpace(version) } -func (p *Provider) getChart(config *releaseConfig, name string, cpo *action.ChartPathOptions) (*chart.Chart, string, error) { +func (p *Provider) getChart(config *ReleaseConfig, name string, cpo *action.ChartPathOptions) (*chart.Chart, string, error) { // TODO: Add a lock as Load function blows up if accessed concurrently path, err := cpo.LocateChart(name, p.cliSettings) diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go new file mode 100644 index 00000000..fc48e80d --- /dev/null +++ b/store/mongodb/provider_repository.go @@ -0,0 +1,53 @@ +package mongodb + +import ( + "context" + "fmt" + "time" + + "github.com/odpf/entropy/store" + + "github.com/odpf/entropy/domain" + "go.mongodb.org/mongo-driver/mongo" +) + +type ProviderRepository struct { + collection *mongo.Collection +} + +func NewProviderRepository(collection *mongo.Collection) *ProviderRepository { + return &ProviderRepository{ + collection: collection, + } +} + +func (rc *ProviderRepository) Migrate() error { + return createUniqueIndex(rc.collection, "urn", 1) +} + +func (rc *ProviderRepository) Create(Provider *domain.Provider) error { + Provider.Urn = domain.GenerateProviderUrn(Provider) + Provider.CreatedAt = time.Now() + Provider.UpdatedAt = time.Now() + + _, err := rc.collection.InsertOne(context.TODO(), Provider) + if err != nil { + if mongo.IsDuplicateKeyError(err) { + return fmt.Errorf("%w: %s", store.ProviderAlreadyExistsError, err) + } + return err + } + return nil +} + +func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { + res := &domain.Provider{} + err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, fmt.Errorf("%w: %s", store.ProviderNotFoundError, err) + } + return nil, err + } + return res, nil +} diff --git a/store/store.go b/store/store.go index 05ae9f5e..c3dac3cc 100644 --- a/store/store.go +++ b/store/store.go @@ -11,13 +11,16 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ResourceAlreadyExistsError = errors.New("resource already exists") - ResourceNotFoundError = errors.New("no resource(s) found") - ModuleAlreadyExistsError = errors.New("module already exists") - ModuleNotFoundError = errors.New("no module(s) found") + ErrResourceAlreadyExists = errors.New("resource already exists") + ErrResourceNotFound = errors.New("no resource(s) found") + ErrModuleAlreadyExists = errors.New("module already exists") + ErrModuleNotFound = errors.New("no module(s) found") + ErrProviderAlreadyExists = errors.New("provider already exists") + ErrProviderNotFound = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" +var ProviderRepositoryName = "providers" type ResourceRepository interface { Create(r *domain.Resource) error @@ -28,6 +31,12 @@ type ResourceRepository interface { Delete(urn string) error } +type ProviderRepository interface { + Create(r *domain.Resource) error + GetByURN(urn string) (*domain.Resource, error) + Migrate() error +} + type ModuleRepository interface { Register(module domain.Module) error Get(id string) (domain.Module, error) From 451930582a4784f29b0beead858bbb555f7c2d86 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 30 Mar 2022 11:39:23 +0530 Subject: [PATCH 02/20] chore: err var name restored --- store/store.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/store/store.go b/store/store.go index c3dac3cc..ab072396 100644 --- a/store/store.go +++ b/store/store.go @@ -11,12 +11,12 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ErrResourceAlreadyExists = errors.New("resource already exists") - ErrResourceNotFound = errors.New("no resource(s) found") - ErrModuleAlreadyExists = errors.New("module already exists") - ErrModuleNotFound = errors.New("no module(s) found") - ErrProviderAlreadyExists = errors.New("provider already exists") - ErrProviderNotFound = errors.New("no provider(s) found") + ResourceAlreadyExistsError = errors.New("resource already exists") + ResourceNotFoundError = errors.New("no resource(s) found") + ModuleAlreadyExistsError = errors.New("module already exists") + ModuleNotFoundError = errors.New("no module(s) found") + ProviderAlreadyExistsError = errors.New("provider already exists") + ProviderNotFoundError = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" From ac5516920068c19773b95eda7a2e82c4f6147388 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 31 Mar 2022 22:21:29 +0530 Subject: [PATCH 03/20] feat: pass providers in resource payload --- app/app.go | 20 ++------------------ domain/provider.go | 3 ++- domain/resource.go | 1 + modules/firehose/module.go | 23 +++++++++++++++++------ store/mongodb/provider_repository.go | 4 ++-- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/app/app.go b/app/app.go index 5ed15bac..3864d2d6 100644 --- a/app/app.go +++ b/app/app.go @@ -10,7 +10,6 @@ import ( "github.com/odpf/entropy/modules/firehose" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/pkg/module" - "github.com/odpf/entropy/pkg/provider/helm" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" "github.com/odpf/entropy/store/inmemory" @@ -35,8 +34,6 @@ import ( "google.golang.org/grpc" ) -var providerURN = "provider_name-kubernetes" - // Config contains the application configuration type Config struct { Service ServiceConfig `mapstructure:"service"` @@ -81,26 +78,13 @@ func RunServer(c *Config) error { ) moduleRepository := inmemory.NewModuleRepository() - err = moduleRepository.Register(log.New(loggerInstance)) - if err != nil { - return err - } - providerFetched, err := providerRepository.GetByURN(providerURN) + err = moduleRepository.Register(log.New(loggerInstance)) if err != nil { return err } - providerConfig := providerFetched.Configs - - kubeConfig := helm.ToKubeConfig(providerConfig) - - helmConfig := &helm.ProviderConfig{ - Kubernetes: kubeConfig, - } - - hp := helm.NewProvider(helmConfig) - err = moduleRepository.Register(firehose.New(hp)) + err = moduleRepository.Register(firehose.New(providerRepository)) if err != nil { return err } diff --git a/domain/provider.go b/domain/provider.go index 3febab12..9b1d30bd 100644 --- a/domain/provider.go +++ b/domain/provider.go @@ -9,6 +9,7 @@ type Provider struct { Urn string `bson:"urn"` Name string `bson:"name"` Kind string `bson:"kind"` + Parent string `bson:"parent"` Configs map[string]interface{} `bson:"configs"` Labels map[string]string `bson:"labels"` CreatedAt time.Time `bson:"created_at"` @@ -17,7 +18,7 @@ type Provider struct { func GenerateProviderUrn(res *Provider) string { return strings.Join([]string{ + sanitizeString(res.Parent), sanitizeString(res.Name), - sanitizeString(res.Kind), }, "-") } diff --git a/domain/resource.go b/domain/resource.go index 09356635..b7609ffb 100644 --- a/domain/resource.go +++ b/domain/resource.go @@ -23,6 +23,7 @@ type Resource struct { Kind string `bson:"kind"` Configs map[string]interface{} `bson:"configs"` Labels map[string]string `bson:"labels"` + Providers map[string]Provider `bson:"providers"` Status ResourceStatus `bson:"status"` CreatedAt time.Time `bson:"created_at"` UpdatedAt time.Time `bson:"updated_at"` diff --git a/modules/firehose/module.go b/modules/firehose/module.go index ca91147b..274d203a 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -9,6 +9,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" "github.com/odpf/entropy/pkg/provider/helm" + "github.com/odpf/entropy/store/mongodb" gjs "github.com/xeipuuv/gojsonschema" ) @@ -248,27 +249,37 @@ const configSchemaString = ` ` type Module struct { - schema *gjs.Schema - helm *helm.Provider + schema *gjs.Schema + providerRepository *mongodb.ProviderRepository } func (m *Module) ID() string { return "firehose" } -func New(helm *helm.Provider) *Module { +func New(providerRepository *mongodb.ProviderRepository) *Module { schemaLoader := gjs.NewStringLoader(configSchemaString) schema, err := gjs.NewSchema(schemaLoader) if err != nil { return nil } return &Module{ - schema: schema, - helm: helm, + schema: schema, + providerRepository: providerRepository, } } func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { + kubeProviderConfig, err := m.providerRepository.GetConfigByURN(r.Providers["kubernetes"].Urn) + if err != nil { + return domain.ResourceStatusError, err + } + kubeConfig := helm.ToKubeConfig(kubeProviderConfig) + helmConfig := &helm.ProviderConfig{ + Kubernetes: kubeConfig, + } + helmProvider := helm.NewProvider(helmConfig) + v := reflect.ValueOf(r.Configs[valuesConfigString]) var values = make(map[string]interface{}) if v.Kind() == reflect.Map { @@ -338,7 +349,7 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { releaseConfig.CreateNamespace = r.Configs[CreateNamespaceConfigString].(bool) } - _, err := m.helm.Release(releaseConfig) + _, err = helmProvider.Release(releaseConfig) if err != nil { return domain.ResourceStatusError, nil } diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go index fc48e80d..b3a04b28 100644 --- a/store/mongodb/provider_repository.go +++ b/store/mongodb/provider_repository.go @@ -40,7 +40,7 @@ func (rc *ProviderRepository) Create(Provider *domain.Provider) error { return nil } -func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { +func (rc *ProviderRepository) GetConfigByURN(urn string) (map[string]interface{}, error) { res := &domain.Provider{} err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) if err != nil { @@ -49,5 +49,5 @@ func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { } return nil, err } - return res, nil + return res.Configs, nil } From 0f13db93930c21bbdb4159ab5e600033c165cd7e Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 4 Apr 2022 20:05:50 +0530 Subject: [PATCH 04/20] feat: add provider service --- api/handlers/v1/{resource.go => server.go} | 127 +++++++++++- .../v1/{resource_test.go => server_test.go} | 30 ++- app/app.go | 14 +- domain/provider.go | 6 +- domain/resource.go | 7 +- go.mod | 2 +- go.sum | 2 + mocks/provider_repository.go | 186 ++++++++++++++++++ mocks/provider_service.go | 118 +++++++++++ modules/firehose/module.go | 107 +++------- pkg/provider/service.go | 46 +++++ .../providers}/helm/kube_rest.go | 0 .../providers}/helm/provider.go | 0 .../providers}/helm/release.go | 0 .../providers}/helm/release_test.go | 0 .../providers}/helm/status.go | 0 store/mongodb/provider_repository.go | 24 ++- store/store.go | 5 +- 18 files changed, 567 insertions(+), 107 deletions(-) rename api/handlers/v1/{resource.go => server.go} (66%) rename api/handlers/v1/{resource_test.go => server_test.go} (95%) create mode 100644 mocks/provider_repository.go create mode 100644 mocks/provider_service.go create mode 100644 pkg/provider/service.go rename {pkg/provider => plugins/providers}/helm/kube_rest.go (100%) rename {pkg/provider => plugins/providers}/helm/provider.go (100%) rename {pkg/provider => plugins/providers}/helm/release.go (100%) rename {pkg/provider => plugins/providers}/helm/release_test.go (100%) rename {pkg/provider => plugins/providers}/helm/status.go (100%) diff --git a/api/handlers/v1/resource.go b/api/handlers/v1/server.go similarity index 66% rename from api/handlers/v1/resource.go rename to api/handlers/v1/server.go index 099d478c..448d18c0 100644 --- a/api/handlers/v1/resource.go +++ b/api/handlers/v1/server.go @@ -3,9 +3,12 @@ package handlersv1 import ( "context" "errors" + "fmt" + "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" "github.com/odpf/entropy/pkg/module" + "github.com/odpf/entropy/pkg/provider" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" entropyv1beta1 "go.buf.build/odpf/gwv/odpf/proton/odpf/entropy/v1beta1" @@ -19,14 +22,17 @@ var ErrInternal = status.Error(codes.Internal, "internal server error") type APIServer struct { entropyv1beta1.UnimplementedResourceServiceServer + entropyv1beta1.UnimplementedProviderServiceServer resourceService resource.ServiceInterface moduleService module.ServiceInterface + providerService provider.ServiceInterface } -func NewApiServer(resourceService resource.ServiceInterface, moduleService module.ServiceInterface) *APIServer { +func NewApiServer(resourceService resource.ServiceInterface, moduleService module.ServiceInterface, providerService provider.ServiceInterface) *APIServer { return &APIServer{ resourceService: resourceService, moduleService: moduleService, + providerService: providerService, } } @@ -178,6 +184,51 @@ func (server APIServer) ApplyAction(ctx context.Context, request *entropyv1beta1 return response, nil } +func (server APIServer) CreateProvider(ctx context.Context, request *entropyv1beta1.CreateProviderRequest) (*entropyv1beta1.CreateProviderResponse, error) { + pro := providerFromProto(request.Provider) + pro.Urn = domain.GenerateProviderUrn(pro) + // TODO: add provider validation + + createdProvider, err := server.providerService.CreateProvider(ctx, pro) + if err != nil { + if errors.Is(err, store.ProviderAlreadyExistsError) { + return nil, status.Error(codes.AlreadyExists, "provider already exists") + } + return nil, ErrInternal + } + + responseProvider, err := providerToProto(createdProvider) + if err != nil { + return nil, ErrInternal + } + response := entropyv1beta1.CreateProviderResponse{ + Provider: responseProvider, + } + return &response, nil +} + +func (server APIServer) ListProviders(ctx context.Context, request *entropyv1beta1.ListProvidersRequest) (*entropyv1beta1.ListProvidersResponse, error) { + var responseProviders []*entropyv1beta1.Provider + providers, err := server.providerService.ListProviders(ctx, request.GetParent(), request.GetKind()) + if err != nil { + return nil, ErrInternal + } + for _, pro := range providers { + responseProvider, err := providerToProto(pro) + if err != nil { + return nil, ErrInternal + } + responseProviders = append(responseProviders, responseProvider) + } + if err != nil { + return nil, ErrInternal + } + response := entropyv1beta1.ListProvidersResponse{ + Providers: responseProviders, + } + return &response, nil +} + func (server APIServer) syncResource(ctx context.Context, updatedResource *domain.Resource) (*domain.Resource, error) { syncedResource, err := server.moduleService.Sync(ctx, updatedResource) if err != nil { @@ -216,12 +267,43 @@ func resourceToProto(res *domain.Resource) (*entropyv1beta1.Resource, error) { Kind: res.Kind, Configs: conf, Labels: res.Labels, + Providers: resourceProvidersToProto(res.Providers), Status: resourceStatusToProto(string(res.Status)), CreatedAt: timestamppb.New(res.CreatedAt), UpdatedAt: timestamppb.New(res.UpdatedAt), }, nil } +func resourceProvidersToProto(ps []domain.ProviderSelector) []*entropyv1beta1.ProviderSelector { + var providerSelectors []*entropyv1beta1.ProviderSelector + + for _, p := range ps { + selector := &entropyv1beta1.ProviderSelector{ + Urn: p.Urn, + Target: p.Target, + } + providerSelectors = append(providerSelectors, selector) + } + return providerSelectors +} + +func providerToProto(pro *domain.Provider) (*entropyv1beta1.Provider, error) { + conf, err := structpb.NewValue(pro.Configs) + if err != nil { + return nil, err + } + return &entropyv1beta1.Provider{ + Urn: pro.Urn, + Name: pro.Name, + Parent: pro.Parent, + Kind: pro.Kind, + Configs: conf, + Labels: pro.Labels, + CreatedAt: timestamppb.New(pro.CreatedAt), + UpdatedAt: timestamppb.New(pro.UpdatedAt), + }, nil +} + func resourceStatusToProto(status string) entropyv1beta1.Resource_Status { if resourceStatus, ok := entropyv1beta1.Resource_Status_value[status]; ok { return entropyv1beta1.Resource_Status(resourceStatus) @@ -231,11 +313,42 @@ func resourceStatusToProto(status string) entropyv1beta1.Resource_Status { func resourceFromProto(res *entropyv1beta1.Resource) *domain.Resource { return &domain.Resource{ - Urn: res.GetUrn(), - Name: res.GetName(), - Parent: res.GetParent(), - Kind: res.GetKind(), - Configs: res.GetConfigs().GetStructValue().AsMap(), - Labels: res.GetLabels(), + Urn: res.GetUrn(), + Name: res.GetName(), + Parent: res.GetParent(), + Kind: res.GetKind(), + Configs: res.GetConfigs().GetStructValue().AsMap(), + Labels: res.GetLabels(), + Providers: providerSelectorFromProto(res.GetProviders()), + } +} + +func providerSelectorFromProto(ps []*entropyv1beta1.ProviderSelector) []domain.ProviderSelector { + var providerSelectors []domain.ProviderSelector + + for _, p := range ps { + selector := domain.ProviderSelector{ + Urn: p.GetUrn(), + Target: p.GetTarget(), + } + providerSelectors = append(providerSelectors, selector) + } + return providerSelectors +} + +func providerFromProto(pro *entropyv1beta1.Provider) *domain.Provider { + var conf map[string]interface{} + err := mapstructure.Decode(pro.GetConfigs(), &conf) + if err != nil { + return nil + } + fmt.Printf("pro.GetConfigs(): %v\n", pro.GetConfigs()) + return &domain.Provider{ + Urn: pro.GetUrn(), + Name: pro.GetName(), + Parent: pro.GetParent(), + Kind: pro.GetKind(), + Configs: conf, + Labels: pro.GetLabels(), } } diff --git a/api/handlers/v1/resource_test.go b/api/handlers/v1/server_test.go similarity index 95% rename from api/handlers/v1/resource_test.go rename to api/handlers/v1/server_test.go index e019ed18..59c80617 100644 --- a/api/handlers/v1/resource_test.go +++ b/api/handlers/v1/server_test.go @@ -102,7 +102,8 @@ func TestAPIServer_CreateResource(t *testing.T) { UpdatedAt: createdAt, }, nil) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -141,7 +142,8 @@ func TestAPIServer_CreateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(nil) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -191,7 +193,8 @@ func TestAPIServer_CreateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ModuleNotFoundError) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -241,7 +244,8 @@ func TestAPIServer_CreateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ModuleConfigParseFailed) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -352,7 +356,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { UpdatedAt: updatedAt, }, nil) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -384,7 +389,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { moduleService := &mocks.ModuleService{} - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -423,7 +429,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ModuleNotFoundError) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -462,7 +469,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ModuleConfigParseFailed) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -620,7 +628,8 @@ func TestAPIServer_DeleteResource(t *testing.T) { moduleService := &mocks.ModuleService{} - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.DeleteResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("DeleteResource() error = %v, wantErr %v", err, wantErr) @@ -644,7 +653,8 @@ func TestAPIServer_DeleteResource(t *testing.T) { moduleService := &mocks.ModuleService{} - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.DeleteResource(ctx, request) if errors.Is(err, nil) { t.Errorf("DeleteResource() got nil error") diff --git a/app/app.go b/app/app.go index 3864d2d6..702c49b0 100644 --- a/app/app.go +++ b/app/app.go @@ -10,6 +10,7 @@ import ( "github.com/odpf/entropy/modules/firehose" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/pkg/module" + "github.com/odpf/entropy/pkg/provider" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" "github.com/odpf/entropy/store/inmemory" @@ -91,6 +92,7 @@ func RunServer(c *Config) error { resourceService := resource.NewService(resourceRepository) moduleService := module.NewService(moduleRepository) + providerService := provider.NewService(providerRepository) muxServer, err := server.NewMux(server.Config{ Port: c.Service.Port, @@ -121,6 +123,11 @@ func RunServer(c *Config) error { return err } + err = gw.RegisterHandler(ctx, entropyv1beta1.RegisterProviderServiceHandlerFromEndpoint) + if err != nil { + return err + } + muxServer.SetGateway("/api", gw) muxServer.RegisterService( @@ -129,7 +136,12 @@ func RunServer(c *Config) error { ) muxServer.RegisterService( &entropyv1beta1.ResourceService_ServiceDesc, - handlersv1.NewApiServer(resourceService, moduleService), + handlersv1.NewApiServer(resourceService, moduleService, providerService), + ) + + muxServer.RegisterService( + &entropyv1beta1.ProviderService_ServiceDesc, + handlersv1.NewApiServer(resourceService, moduleService, providerService), ) muxServer.RegisterHandler("/ping", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/domain/provider.go b/domain/provider.go index 9b1d30bd..5159678e 100644 --- a/domain/provider.go +++ b/domain/provider.go @@ -16,9 +16,9 @@ type Provider struct { UpdatedAt time.Time `bson:"updated_at"` } -func GenerateProviderUrn(res *Provider) string { +func GenerateProviderUrn(pro *Provider) string { return strings.Join([]string{ - sanitizeString(res.Parent), - sanitizeString(res.Name), + sanitizeString(pro.Parent), + sanitizeString(pro.Name), }, "-") } diff --git a/domain/resource.go b/domain/resource.go index b7609ffb..9871d74a 100644 --- a/domain/resource.go +++ b/domain/resource.go @@ -16,6 +16,11 @@ const ( ResourceStatusCompleted ResourceStatus = "STATUS_COMPLETED" ) +type ProviderSelector struct { + Urn string `bson:"urn"` + Target string `bson:"target"` +} + type Resource struct { Urn string `bson:"urn"` Name string `bson:"name"` @@ -23,7 +28,7 @@ type Resource struct { Kind string `bson:"kind"` Configs map[string]interface{} `bson:"configs"` Labels map[string]string `bson:"labels"` - Providers map[string]Provider `bson:"providers"` + Providers []ProviderSelector `bson:"providers"` Status ResourceStatus `bson:"status"` CreatedAt time.Time `bson:"created_at"` UpdatedAt time.Time `bson:"updated_at"` diff --git a/go.mod b/go.mod index db23b559..17c99489 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xeipuuv/gojsonschema v1.2.0 go.buf.build/odpf/gw/odpf/proton v1.1.9 - go.buf.build/odpf/gwv/odpf/proton v1.1.65 + go.buf.build/odpf/gwv/odpf/proton v1.1.76 go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.19.0 google.golang.org/grpc v1.43.0 diff --git a/go.sum b/go.sum index 89c2b7bc..9dabe755 100644 --- a/go.sum +++ b/go.sum @@ -1164,6 +1164,8 @@ go.buf.build/odpf/gwv/envoyproxy/protoc-gen-validate v1.1.3/go.mod h1:2Tg6rYIoDh go.buf.build/odpf/gwv/grpc-ecosystem/grpc-gateway v1.1.37/go.mod h1:UrBCdmHgaY/pLapYUMOq01c1yuzwT8AEBTsgpmzq2zo= go.buf.build/odpf/gwv/odpf/proton v1.1.65 h1:A0jh+7kMfiGWoSXpIoMOpO1sDTETLEo09y2wY7e40ZA= go.buf.build/odpf/gwv/odpf/proton v1.1.65/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= +go.buf.build/odpf/gwv/odpf/proton v1.1.76 h1:F8WbVUSHwGI1wHOewbLVIcshEJSJ/3HU6VKmSRNqtL0= +go.buf.build/odpf/gwv/odpf/proton v1.1.76/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= diff --git a/mocks/provider_repository.go b/mocks/provider_repository.go new file mode 100644 index 00000000..ea79ec63 --- /dev/null +++ b/mocks/provider_repository.go @@ -0,0 +1,186 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + domain "github.com/odpf/entropy/domain" + mock "github.com/stretchr/testify/mock" +) + +// ProviderRepository is an autogenerated mock type for the ProviderRepository type +type ProviderRepository struct { + mock.Mock +} + +type ProviderRepository_Expecter struct { + mock *mock.Mock +} + +func (_m *ProviderRepository) EXPECT() *ProviderRepository_Expecter { + return &ProviderRepository_Expecter{mock: &_m.Mock} +} + +// Create provides a mock function with given fields: r +func (_m *ProviderRepository) Create(r *domain.Provider) error { + ret := _m.Called(r) + + var r0 error + if rf, ok := ret.Get(0).(func(*domain.Provider) error); ok { + r0 = rf(r) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ProviderRepository_Create_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Create' +type ProviderRepository_Create_Call struct { + *mock.Call +} + +// Create is a helper method to define mock.On call +// - r *domain.Provider +func (_e *ProviderRepository_Expecter) Create(r interface{}) *ProviderRepository_Create_Call { + return &ProviderRepository_Create_Call{Call: _e.mock.On("Create", r)} +} + +func (_c *ProviderRepository_Create_Call) Run(run func(r *domain.Provider)) *ProviderRepository_Create_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*domain.Provider)) + }) + return _c +} + +func (_c *ProviderRepository_Create_Call) Return(_a0 error) *ProviderRepository_Create_Call { + _c.Call.Return(_a0) + return _c +} + +// GetByURN provides a mock function with given fields: urn +func (_m *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { + ret := _m.Called(urn) + + var r0 *domain.Provider + if rf, ok := ret.Get(0).(func(string) *domain.Provider); ok { + r0 = rf(urn) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(urn) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderRepository_GetByURN_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetByURN' +type ProviderRepository_GetByURN_Call struct { + *mock.Call +} + +// GetByURN is a helper method to define mock.On call +// - urn string +func (_e *ProviderRepository_Expecter) GetByURN(urn interface{}) *ProviderRepository_GetByURN_Call { + return &ProviderRepository_GetByURN_Call{Call: _e.mock.On("GetByURN", urn)} +} + +func (_c *ProviderRepository_GetByURN_Call) Run(run func(urn string)) *ProviderRepository_GetByURN_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ProviderRepository_GetByURN_Call) Return(_a0 *domain.Provider, _a1 error) *ProviderRepository_GetByURN_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// List provides a mock function with given fields: filter +func (_m *ProviderRepository) List(filter map[string]string) ([]*domain.Provider, error) { + ret := _m.Called(filter) + + var r0 []*domain.Provider + if rf, ok := ret.Get(0).(func(map[string]string) []*domain.Provider); ok { + r0 = rf(filter) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(map[string]string) error); ok { + r1 = rf(filter) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderRepository_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type ProviderRepository_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - filter map[string]string +func (_e *ProviderRepository_Expecter) List(filter interface{}) *ProviderRepository_List_Call { + return &ProviderRepository_List_Call{Call: _e.mock.On("List", filter)} +} + +func (_c *ProviderRepository_List_Call) Run(run func(filter map[string]string)) *ProviderRepository_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(map[string]string)) + }) + return _c +} + +func (_c *ProviderRepository_List_Call) Return(_a0 []*domain.Provider, _a1 error) *ProviderRepository_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// Migrate provides a mock function with given fields: +func (_m *ProviderRepository) Migrate() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ProviderRepository_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate' +type ProviderRepository_Migrate_Call struct { + *mock.Call +} + +// Migrate is a helper method to define mock.On call +func (_e *ProviderRepository_Expecter) Migrate() *ProviderRepository_Migrate_Call { + return &ProviderRepository_Migrate_Call{Call: _e.mock.On("Migrate")} +} + +func (_c *ProviderRepository_Migrate_Call) Run(run func()) *ProviderRepository_Migrate_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ProviderRepository_Migrate_Call) Return(_a0 error) *ProviderRepository_Migrate_Call { + _c.Call.Return(_a0) + return _c +} diff --git a/mocks/provider_service.go b/mocks/provider_service.go new file mode 100644 index 00000000..dcfa87bc --- /dev/null +++ b/mocks/provider_service.go @@ -0,0 +1,118 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + domain "github.com/odpf/entropy/domain" + mock "github.com/stretchr/testify/mock" +) + +// ProviderService is an autogenerated mock type for the ServiceInterface type +type ProviderService struct { + mock.Mock +} + +type ProviderService_Expecter struct { + mock *mock.Mock +} + +func (_m *ProviderService) EXPECT() *ProviderService_Expecter { + return &ProviderService_Expecter{mock: &_m.Mock} +} + +// CreateProvider provides a mock function with given fields: ctx, res +func (_m *ProviderService) CreateProvider(ctx context.Context, res *domain.Provider) (*domain.Provider, error) { + ret := _m.Called(ctx, res) + + var r0 *domain.Provider + if rf, ok := ret.Get(0).(func(context.Context, *domain.Provider) *domain.Provider); ok { + r0 = rf(ctx, res) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *domain.Provider) error); ok { + r1 = rf(ctx, res) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderService_CreateProvider_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateProvider' +type ProviderService_CreateProvider_Call struct { + *mock.Call +} + +// CreateProvider is a helper method to define mock.On call +// - ctx context.Context +// - res *domain.Provider +func (_e *ProviderService_Expecter) CreateProvider(ctx interface{}, res interface{}) *ProviderService_CreateProvider_Call { + return &ProviderService_CreateProvider_Call{Call: _e.mock.On("CreateProvider", ctx, res)} +} + +func (_c *ProviderService_CreateProvider_Call) Run(run func(ctx context.Context, res *domain.Provider)) *ProviderService_CreateProvider_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*domain.Provider)) + }) + return _c +} + +func (_c *ProviderService_CreateProvider_Call) Return(_a0 *domain.Provider, _a1 error) *ProviderService_CreateProvider_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ListProviders provides a mock function with given fields: ctx, parent, kind +func (_m *ProviderService) ListProviders(ctx context.Context, parent string, kind string) ([]*domain.Provider, error) { + ret := _m.Called(ctx, parent, kind) + + var r0 []*domain.Provider + if rf, ok := ret.Get(0).(func(context.Context, string, string) []*domain.Provider); ok { + r0 = rf(ctx, parent, kind) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, parent, kind) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderService_ListProviders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListProviders' +type ProviderService_ListProviders_Call struct { + *mock.Call +} + +// ListProviders is a helper method to define mock.On call +// - ctx context.Context +// - parent string +// - kind string +func (_e *ProviderService_Expecter) ListProviders(ctx interface{}, parent interface{}, kind interface{}) *ProviderService_ListProviders_Call { + return &ProviderService_ListProviders_Call{Call: _e.mock.On("ListProviders", ctx, parent, kind)} +} + +func (_c *ProviderService_ListProviders_Call) Run(run func(ctx context.Context, parent string, kind string)) *ProviderService_ListProviders_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *ProviderService_ListProviders_Call) Return(_a0 []*domain.Provider, _a1 error) *ProviderService_ListProviders_Call { + _c.Call.Return(_a0, _a1) + return _c +} diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 274d203a..5596cd35 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -8,7 +8,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" - "github.com/odpf/entropy/pkg/provider/helm" + "github.com/odpf/entropy/plugins/providers/helm" "github.com/odpf/entropy/store/mongodb" gjs "github.com/xeipuuv/gojsonschema" ) @@ -28,6 +28,7 @@ const ( replaceConfigString = "replace" descriptionConfigString = "description" CreateNamespaceConfigString = "create_namespace" + KUBERNETES = "kubernetes" ) const configSchemaString = ` @@ -270,95 +271,45 @@ func New(providerRepository *mongodb.ProviderRepository) *Module { } func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { - kubeProviderConfig, err := m.providerRepository.GetConfigByURN(r.Providers["kubernetes"].Urn) - if err != nil { - return domain.ResourceStatusError, err - } - kubeConfig := helm.ToKubeConfig(kubeProviderConfig) - helmConfig := &helm.ProviderConfig{ - Kubernetes: kubeConfig, - } - helmProvider := helm.NewProvider(helmConfig) - - v := reflect.ValueOf(r.Configs[valuesConfigString]) - var values = make(map[string]interface{}) - if v.Kind() == reflect.Map { - for _, key := range v.MapKeys() { - strct := v.MapIndex(key) - values[key.String()] = strct.Interface() + for _, p := range r.Providers { + provider, err := m.providerRepository.GetByURN(p.Urn) + if err != nil { + return domain.ResourceStatusError, err } - } - - var rc helm.ReleaseConfig - if err := mapstructure.Decode(r.Configs, &rc); err != nil { - return domain.ResourceStatusError, err - } - - releaseConfig := helm.DefaultReleaseConfig() - releaseConfig.Values = values - - if r.Configs[nameConfigString] != nil { - releaseConfig.Name = r.Configs[nameConfigString].(string) - } - - if r.Configs[repositoryConfigString] != nil { - releaseConfig.Repository = r.Configs[repositoryConfigString].(string) - } - - if r.Configs[chartConfigString] != nil { - releaseConfig.Chart = r.Configs[chartConfigString].(string) - } - - if r.Configs[versionConfigString] != nil { - releaseConfig.Version = r.Configs[versionConfigString].(string) - } - - if r.Configs[namespaceConfigString] != nil { - releaseConfig.Namespace = r.Configs[namespaceConfigString].(string) - } - if r.Configs[timeoutConfigString] != nil { - releaseConfig.Timeout = r.Configs[timeoutConfigString].(int) - } - - if r.Configs[descriptionConfigString] != nil { - releaseConfig.Description = r.Configs[descriptionConfigString].(string) - } - - if r.Configs[forceUpdateConfigString] != nil { - releaseConfig.ForceUpdate = r.Configs[forceUpdateConfigString].(bool) - } - - if r.Configs[recreatePodsConfigString] != nil { - releaseConfig.RecreatePods = r.Configs[recreatePodsConfigString].(bool) - } - - if r.Configs[waitConfigString] != nil { - releaseConfig.Wait = r.Configs[waitConfigString].(bool) - } + if provider.Kind == KUBERNETES { + releaseConfig := helm.DefaultReleaseConfig() + var values = make(map[string]interface{}) - if r.Configs[waitForJobsConfigString] != nil { - releaseConfig.WaitForJobs = r.Configs[waitForJobsConfigString].(bool) - } + kubeConfig := helm.ToKubeConfig(provider.Configs) + helmConfig := &helm.ProviderConfig{ + Kubernetes: kubeConfig, + } + helmProvider := helm.NewProvider(helmConfig) - if r.Configs[replaceConfigString] != nil { - releaseConfig.Replace = r.Configs[replaceConfigString].(bool) - } + v := reflect.ValueOf(r.Configs[valuesConfigString]) + if err := mapstructure.Decode(v, &values); err != nil { + return domain.ResourceStatusError, err + } + releaseConfig.Values = values - if r.Configs[CreateNamespaceConfigString] != nil { - releaseConfig.CreateNamespace = r.Configs[CreateNamespaceConfigString].(bool) - } + err := mapstructure.Decode(r.Configs, &releaseConfig) + if err != nil { + return domain.ResourceStatusError, err + } - _, err = helmProvider.Release(releaseConfig) - if err != nil { - return domain.ResourceStatusError, nil + _, err = helmProvider.Release(releaseConfig) + if err != nil { + return domain.ResourceStatusError, nil + } + } } return domain.ResourceStatusCompleted, nil } func (m *Module) Validate(r *domain.Resource) error { - resourceLoader := gjs.NewGoLoader(r.Configs["values"]) + resourceLoader := gjs.NewGoLoader(r.Configs[valuesConfigString]) result, err := m.schema.Validate(resourceLoader) if err != nil { return fmt.Errorf("%w: %s", domain.ModuleConfigParseFailed, err) diff --git a/pkg/provider/service.go b/pkg/provider/service.go new file mode 100644 index 00000000..e327f2f2 --- /dev/null +++ b/pkg/provider/service.go @@ -0,0 +1,46 @@ +package provider + +import ( + "context" + + "github.com/odpf/entropy/domain" + "github.com/odpf/entropy/store" +) + +type ServiceInterface interface { + CreateProvider(ctx context.Context, res *domain.Provider) (*domain.Provider, error) + ListProviders(ctx context.Context, parent string, kind string) ([]*domain.Provider, error) +} + +type Service struct { + providerRepository store.ProviderRepository +} + +func NewService(repository store.ProviderRepository) *Service { + return &Service{ + providerRepository: repository, + } +} + +func (s *Service) CreateProvider(ctx context.Context, pro *domain.Provider) (*domain.Provider, error) { + err := s.providerRepository.Create(pro) + if err != nil { + return nil, err + } + createdProvider, err := s.providerRepository.GetByURN(pro.Urn) + if err != nil { + return nil, err + } + return createdProvider, nil +} + +func (s *Service) ListProviders(ctx context.Context, parent string, kind string) ([]*domain.Provider, error) { + filter := map[string]string{} + if kind != "" { + filter["kind"] = kind + } + if parent != "" { + filter["parent"] = parent + } + return s.providerRepository.List(filter) +} diff --git a/pkg/provider/helm/kube_rest.go b/plugins/providers/helm/kube_rest.go similarity index 100% rename from pkg/provider/helm/kube_rest.go rename to plugins/providers/helm/kube_rest.go diff --git a/pkg/provider/helm/provider.go b/plugins/providers/helm/provider.go similarity index 100% rename from pkg/provider/helm/provider.go rename to plugins/providers/helm/provider.go diff --git a/pkg/provider/helm/release.go b/plugins/providers/helm/release.go similarity index 100% rename from pkg/provider/helm/release.go rename to plugins/providers/helm/release.go diff --git a/pkg/provider/helm/release_test.go b/plugins/providers/helm/release_test.go similarity index 100% rename from pkg/provider/helm/release_test.go rename to plugins/providers/helm/release_test.go diff --git a/pkg/provider/helm/status.go b/plugins/providers/helm/status.go similarity index 100% rename from pkg/provider/helm/status.go rename to plugins/providers/helm/status.go diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go index b3a04b28..a73c9ee7 100644 --- a/store/mongodb/provider_repository.go +++ b/store/mongodb/provider_repository.go @@ -40,14 +40,30 @@ func (rc *ProviderRepository) Create(Provider *domain.Provider) error { return nil } -func (rc *ProviderRepository) GetConfigByURN(urn string) (map[string]interface{}, error) { - res := &domain.Provider{} - err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) +func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { + pro := &domain.Provider{} + err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(pro) if err != nil { if err == mongo.ErrNoDocuments { return nil, fmt.Errorf("%w: %s", store.ProviderNotFoundError, err) } return nil, err } - return res.Configs, nil + return pro, nil +} + +func (rc *ProviderRepository) List(filter map[string]string) ([]*domain.Provider, error) { + var pro []*domain.Provider + cur, err := rc.collection.Find(context.TODO(), filter) + if err != nil { + return nil, err + } + err = cur.All(context.TODO(), &pro) + if err != nil { + if err == mongo.ErrNoDocuments { + return pro, nil + } + return nil, err + } + return pro, nil } diff --git a/store/store.go b/store/store.go index ab072396..4860a559 100644 --- a/store/store.go +++ b/store/store.go @@ -32,8 +32,9 @@ type ResourceRepository interface { } type ProviderRepository interface { - Create(r *domain.Resource) error - GetByURN(urn string) (*domain.Resource, error) + Create(r *domain.Provider) error + GetByURN(urn string) (*domain.Provider, error) + List(filter map[string]string) ([]*domain.Provider, error) Migrate() error } From 3fd60ab999743f7829c9440ee8321bdcb99c3ebe Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 4 Apr 2022 20:22:01 +0530 Subject: [PATCH 05/20] chore: remove unused constants --- go.sum | 2 -- modules/firehose/module.go | 17 ++--------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/go.sum b/go.sum index 9dabe755..770e44b8 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,6 @@ go.buf.build/odpf/gw/odpf/proton v1.1.9 h1:iEdRUVVc/HwOqB7WRXjhXjR2pza2gyUbQ74G2 go.buf.build/odpf/gw/odpf/proton v1.1.9/go.mod h1:I9E8CF7w/690vRNWqBU6qDcUbi3Pi2THdn1yycBVTDQ= go.buf.build/odpf/gwv/envoyproxy/protoc-gen-validate v1.1.3/go.mod h1:2Tg6rYIoDhpl39Zd2+WBOF9uG4XxAOs0bK2Z2/bwTOc= go.buf.build/odpf/gwv/grpc-ecosystem/grpc-gateway v1.1.37/go.mod h1:UrBCdmHgaY/pLapYUMOq01c1yuzwT8AEBTsgpmzq2zo= -go.buf.build/odpf/gwv/odpf/proton v1.1.65 h1:A0jh+7kMfiGWoSXpIoMOpO1sDTETLEo09y2wY7e40ZA= -go.buf.build/odpf/gwv/odpf/proton v1.1.65/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.buf.build/odpf/gwv/odpf/proton v1.1.76 h1:F8WbVUSHwGI1wHOewbLVIcshEJSJ/3HU6VKmSRNqtL0= go.buf.build/odpf/gwv/odpf/proton v1.1.76/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 5596cd35..766baacb 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -14,21 +14,8 @@ import ( ) const ( - nameConfigString = "name" - repositoryConfigString = "repository" - chartConfigString = "chart" - versionConfigString = "version" - valuesConfigString = "values" - namespaceConfigString = "namespace" - timeoutConfigString = "timeout" - forceUpdateConfigString = "force_update" - recreatePodsConfigString = "recreate_pods" - waitConfigString = "wait" - waitForJobsConfigString = "wait_for_jobs" - replaceConfigString = "replace" - descriptionConfigString = "description" - CreateNamespaceConfigString = "create_namespace" - KUBERNETES = "kubernetes" + valuesConfigString = "values" + KUBERNETES = "kubernetes" ) const configSchemaString = ` From d9789c3b65968ea6595ad72ca8329b72adc5e7fd Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 4 Apr 2022 20:46:00 +0530 Subject: [PATCH 06/20] chore: error variable name changed (#28) * chore: error variable name changed as per golang convention * fix: change error name ErrModuleConfigParseFailed --- api/handlers/v1/resource.go | 14 +++++++------- api/handlers/v1/resource_test.go | 18 +++++++++--------- domain/module.go | 2 +- modules/firehose/module.go | 2 +- modules/log/module.go | 5 +++-- pkg/module/service_test.go | 15 ++++++++------- pkg/resource/service_test.go | 19 ++++++++++--------- store/inmemory/module_repository.go | 4 ++-- store/inmemory/module_repository_test.go | 9 +++++---- store/mongodb/resource_repository.go | 8 ++++---- store/mongodb/resource_repository_test.go | 11 ++++++----- store/store.go | 8 ++++---- 12 files changed, 60 insertions(+), 55 deletions(-) diff --git a/api/handlers/v1/resource.go b/api/handlers/v1/resource.go index 099d478c..b70240af 100644 --- a/api/handlers/v1/resource.go +++ b/api/handlers/v1/resource.go @@ -39,7 +39,7 @@ func (server APIServer) CreateResource(ctx context.Context, request *entropyv1be } createdResource, err := server.resourceService.CreateResource(ctx, res) if err != nil { - if errors.Is(err, store.ResourceAlreadyExistsError) { + if errors.Is(err, store.ErrResourceAlreadyExists) { return nil, status.Error(codes.AlreadyExists, "resource already exists") } return nil, ErrInternal @@ -61,7 +61,7 @@ func (server APIServer) CreateResource(ctx context.Context, request *entropyv1be func (server APIServer) UpdateResource(ctx context.Context, request *entropyv1beta1.UpdateResourceRequest) (*entropyv1beta1.UpdateResourceResponse, error) { res, err := server.resourceService.GetResource(ctx, request.GetUrn()) if err != nil { - if errors.Is(err, store.ResourceNotFoundError) { + if errors.Is(err, store.ErrResourceNotFound) { return nil, status.Error(codes.NotFound, "could not find resource with given urn") } return nil, ErrInternal @@ -93,7 +93,7 @@ func (server APIServer) UpdateResource(ctx context.Context, request *entropyv1be func (server APIServer) GetResource(ctx context.Context, request *entropyv1beta1.GetResourceRequest) (*entropyv1beta1.GetResourceResponse, error) { res, err := server.resourceService.GetResource(ctx, request.GetUrn()) if err != nil { - if errors.Is(err, store.ResourceNotFoundError) { + if errors.Is(err, store.ErrResourceNotFound) { return nil, status.Error(codes.NotFound, "could not find resource with given urn") } return nil, ErrInternal @@ -134,7 +134,7 @@ func (server APIServer) DeleteResource(ctx context.Context, request *entropyv1be urn := request.GetUrn() _, err := server.resourceService.GetResource(ctx, urn) if err != nil { - if errors.Is(err, store.ResourceNotFoundError) { + if errors.Is(err, store.ErrResourceNotFound) { return nil, status.Error(codes.NotFound, "could not find resource with given urn") } return nil, ErrInternal @@ -152,7 +152,7 @@ func (server APIServer) DeleteResource(ctx context.Context, request *entropyv1be func (server APIServer) ApplyAction(ctx context.Context, request *entropyv1beta1.ApplyActionRequest) (*entropyv1beta1.ApplyActionResponse, error) { res, err := server.resourceService.GetResource(ctx, request.GetUrn()) if err != nil { - if errors.Is(err, store.ResourceNotFoundError) { + if errors.Is(err, store.ErrResourceNotFound) { return nil, status.Error(codes.NotFound, "could not find resource with given urn") } return nil, ErrInternal @@ -193,10 +193,10 @@ func (server APIServer) syncResource(ctx context.Context, updatedResource *domai func (server APIServer) validateResource(ctx context.Context, res *domain.Resource) error { err := server.moduleService.Validate(ctx, res) if err != nil { - if errors.Is(err, store.ModuleNotFoundError) { + if errors.Is(err, store.ErrModuleNotFound) { return status.Errorf(codes.InvalidArgument, "failed to find module to deploy this kind") } - if errors.Is(err, domain.ModuleConfigParseFailed) { + if errors.Is(err, domain.ErrModuleConfigParseFailed) { return status.Errorf(codes.InvalidArgument, "failed to parse configs") } return status.Errorf(codes.InvalidArgument, err.Error()) diff --git a/api/handlers/v1/resource_test.go b/api/handlers/v1/resource_test.go index e019ed18..8ab81be4 100644 --- a/api/handlers/v1/resource_test.go +++ b/api/handlers/v1/resource_test.go @@ -135,7 +135,7 @@ func TestAPIServer_CreateResource(t *testing.T) { resourceService.EXPECT(). CreateResource(mock.Anything, mock.Anything). - Return(nil, store.ResourceAlreadyExistsError). + Return(nil, store.ErrResourceAlreadyExists). Once() moduleService := &mocks.ModuleService{} @@ -189,7 +189,7 @@ func TestAPIServer_CreateResource(t *testing.T) { }, nil).Once() moduleService := &mocks.ModuleService{} - moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ModuleNotFoundError) + moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ErrModuleNotFound) server := NewApiServer(resourceService, moduleService) got, err := server.CreateResource(ctx, request) @@ -239,7 +239,7 @@ func TestAPIServer_CreateResource(t *testing.T) { }, nil).Once() moduleService := &mocks.ModuleService{} - moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ModuleConfigParseFailed) + moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ErrModuleConfigParseFailed) server := NewApiServer(resourceService, moduleService) got, err := server.CreateResource(ctx, request) @@ -380,7 +380,7 @@ func TestAPIServer_UpdateResource(t *testing.T) { resourceService.EXPECT(). GetResource(mock.Anything, mock.Anything). - Return(nil, store.ResourceNotFoundError).Once() + Return(nil, store.ErrResourceNotFound).Once() moduleService := &mocks.ModuleService{} @@ -421,7 +421,7 @@ func TestAPIServer_UpdateResource(t *testing.T) { }, nil).Once() moduleService := &mocks.ModuleService{} - moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ModuleNotFoundError) + moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ErrModuleNotFound) server := NewApiServer(resourceService, moduleService) got, err := server.UpdateResource(ctx, request) @@ -460,7 +460,7 @@ func TestAPIServer_UpdateResource(t *testing.T) { }, nil).Once() moduleService := &mocks.ModuleService{} - moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ModuleConfigParseFailed) + moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ErrModuleConfigParseFailed) server := NewApiServer(resourceService, moduleService) got, err := server.UpdateResource(ctx, request) @@ -523,7 +523,7 @@ func TestAPIServer_GetResource(t *testing.T) { wantErr := status.Error(codes.NotFound, "could not find resource with given urn") mockResourceService := &mocks.ResourceService{} - mockResourceService.EXPECT().GetResource(mock.Anything, mock.Anything).Return(nil, store.ResourceNotFoundError).Once() + mockResourceService.EXPECT().GetResource(mock.Anything, mock.Anything).Return(nil, store.ErrResourceNotFound).Once() mockModuleService := &mocks.ModuleService{} @@ -640,7 +640,7 @@ func TestAPIServer_DeleteResource(t *testing.T) { resourceService := &mocks.ResourceService{} resourceService.EXPECT(). GetResource(mock.Anything, "p-testdata-gl-testname-log"). - Return(nil, store.ResourceNotFoundError).Once() + Return(nil, store.ErrResourceNotFound).Once() moduleService := &mocks.ModuleService{} @@ -751,7 +751,7 @@ func TestAPIServer_ApplyAction(t *testing.T) { resourceService := &mocks.ResourceService{} resourceService.EXPECT(). GetResource(mock.Anything, "p-testdata-gl-testname-log"). - Return(nil, store.ResourceNotFoundError).Once() + Return(nil, store.ErrResourceNotFound).Once() moduleService := &mocks.ModuleService{} diff --git a/domain/module.go b/domain/module.go index 22f3e925..e03b4bc8 100644 --- a/domain/module.go +++ b/domain/module.go @@ -5,7 +5,7 @@ package domain import "errors" var ( - ModuleConfigParseFailed = errors.New("unable to load and validate config") + ErrModuleConfigParseFailed = errors.New("unable to load and validate config") ) type Module interface { diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 893f44f0..a9627f74 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -254,7 +254,7 @@ func (m *Module) Validate(r *domain.Resource) error { resourceLoader := gjs.NewGoLoader(r.Configs) result, err := m.schema.Validate(resourceLoader) if err != nil { - return fmt.Errorf("%w: %s", domain.ModuleConfigParseFailed, err) + return fmt.Errorf("%w: %s", domain.ErrModuleConfigParseFailed, err) } if !result.Valid() { var errorStrings []string diff --git a/modules/log/module.go b/modules/log/module.go index a034771a..90e39338 100644 --- a/modules/log/module.go +++ b/modules/log/module.go @@ -3,10 +3,11 @@ package log import ( "errors" "fmt" + "strings" + "github.com/odpf/entropy/domain" gjs "github.com/xeipuuv/gojsonschema" "go.uber.org/zap" - "strings" ) type Level string @@ -77,7 +78,7 @@ func (m *Module) Validate(r *domain.Resource) error { resourceLoader := gjs.NewGoLoader(r.Configs) result, err := m.schema.Validate(resourceLoader) if err != nil { - return fmt.Errorf("%w: %s", domain.ModuleConfigParseFailed, err) + return fmt.Errorf("%w: %s", domain.ErrModuleConfigParseFailed, err) } if !result.Valid() { var errorStrings []string diff --git a/pkg/module/service_test.go b/pkg/module/service_test.go index ca30a7d2..704ee383 100644 --- a/pkg/module/service_test.go +++ b/pkg/module/service_test.go @@ -3,13 +3,14 @@ package module import ( "context" "errors" + "reflect" + "testing" + "time" + "github.com/odpf/entropy/domain" "github.com/odpf/entropy/mocks" "github.com/odpf/entropy/store" "github.com/stretchr/testify/mock" - "reflect" - "testing" - "time" ) func TestService_Sync(t *testing.T) { @@ -75,7 +76,7 @@ func TestService_Sync(t *testing.T) { { name: "test sync module not found error", setup: func(t *testing.T) { - mockModuleRepo.EXPECT().Get("mock").Return(nil, store.ModuleNotFoundError).Once() + mockModuleRepo.EXPECT().Get("mock").Return(nil, store.ErrModuleNotFound).Once() }, fields: fields{ moduleRepository: mockModuleRepo, @@ -95,7 +96,7 @@ func TestService_Sync(t *testing.T) { CreatedAt: currentTime, UpdatedAt: currentTime, }, - wantErr: store.ModuleNotFoundError, + wantErr: store.ErrModuleNotFound, }, { name: "test sync module error while applying", @@ -195,7 +196,7 @@ func TestService_Validate(t *testing.T) { { name: "test validate module not found error", setup: func(t *testing.T) { - mockModuleRepo.EXPECT().Get("mock").Return(nil, store.ModuleNotFoundError).Once() + mockModuleRepo.EXPECT().Get("mock").Return(nil, store.ErrModuleNotFound).Once() }, fields: fields{ moduleRepository: mockModuleRepo, @@ -204,7 +205,7 @@ func TestService_Validate(t *testing.T) { ctx: nil, r: r, }, - wantErr: store.ModuleNotFoundError, + wantErr: store.ErrModuleNotFound, }, { name: "test validation failed", diff --git a/pkg/resource/service_test.go b/pkg/resource/service_test.go index eebc2420..287b60df 100644 --- a/pkg/resource/service_test.go +++ b/pkg/resource/service_test.go @@ -3,14 +3,15 @@ package resource import ( "context" "errors" + "reflect" + "testing" + "time" + "github.com/odpf/entropy/domain" "github.com/odpf/entropy/mocks" "github.com/odpf/entropy/store" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "reflect" - "testing" - "time" ) func TestService_CreateResource(t *testing.T) { @@ -76,10 +77,10 @@ func TestService_CreateResource(t *testing.T) { Labels: map[string]string{}, } want := (*domain.Resource)(nil) - wantErr := store.ResourceAlreadyExistsError + wantErr := store.ErrResourceAlreadyExists mockRepo.EXPECT().Create(mock.Anything).Run(func(r *domain.Resource) { assert.Equal(t, domain.ResourceStatusPending, r.Status) - }).Return(store.ResourceAlreadyExistsError).Once() + }).Return(store.ErrResourceAlreadyExists).Once() s := NewService(mockRepo) got, err := s.CreateResource(context.Background(), argResource) @@ -158,11 +159,11 @@ func TestService_UpdateResource(t *testing.T) { mockRepo := &mocks.ResourceRepository{} want := (*domain.Resource)(nil) - wantErr := store.ResourceNotFoundError + wantErr := store.ErrResourceNotFound mockRepo.EXPECT(). Update(mock.Anything). - Return(store.ResourceNotFoundError). + Return(store.ErrResourceNotFound). Once() s := NewService(mockRepo) @@ -236,11 +237,11 @@ func TestService_GetResource(t *testing.T) { mockRepo := &mocks.ResourceRepository{} want := (*domain.Resource)(nil) - wantErr := store.ResourceNotFoundError + wantErr := store.ErrResourceNotFound mockRepo.EXPECT(). GetByURN(mock.Anything). - Return(nil, store.ResourceNotFoundError). + Return(nil, store.ErrResourceNotFound). Once() s := NewService(mockRepo) diff --git a/store/inmemory/module_repository.go b/store/inmemory/module_repository.go index c3a7742b..923844ad 100644 --- a/store/inmemory/module_repository.go +++ b/store/inmemory/module_repository.go @@ -17,7 +17,7 @@ func NewModuleRepository() *ModuleRepository { func (mr *ModuleRepository) Register(module domain.Module) error { if _, exists := mr.collection[module.ID()]; exists { - return store.ModuleAlreadyExistsError + return store.ErrModuleAlreadyExists } mr.collection[module.ID()] = module return nil @@ -27,5 +27,5 @@ func (mr *ModuleRepository) Get(id string) (domain.Module, error) { if module, exists := mr.collection[id]; exists { return module, nil } - return nil, store.ModuleNotFoundError + return nil, store.ErrModuleNotFound } diff --git a/store/inmemory/module_repository_test.go b/store/inmemory/module_repository_test.go index 8e53ea28..cb6fbbae 100644 --- a/store/inmemory/module_repository_test.go +++ b/store/inmemory/module_repository_test.go @@ -2,11 +2,12 @@ package inmemory import ( "errors" + "reflect" + "testing" + "github.com/odpf/entropy/domain" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/store" - "reflect" - "testing" ) func TestModuleRepository_Get(t *testing.T) { @@ -48,7 +49,7 @@ func TestModuleRepository_Get(t *testing.T) { id: "notlog", }, want: nil, - wantErr: store.ModuleNotFoundError, + wantErr: store.ErrModuleNotFound, }, } for _, tt := range tests { @@ -102,7 +103,7 @@ func TestModuleRepository_Register(t *testing.T) { args: args{ module: mod, }, - wantErr: store.ModuleAlreadyExistsError, + wantErr: store.ErrModuleAlreadyExists, }, } for _, tt := range tests { diff --git a/store/mongodb/resource_repository.go b/store/mongodb/resource_repository.go index aaf3d80f..6f351353 100644 --- a/store/mongodb/resource_repository.go +++ b/store/mongodb/resource_repository.go @@ -45,7 +45,7 @@ func (rc *ResourceRepository) Create(resource *domain.Resource) error { _, err := rc.collection.InsertOne(context.TODO(), resource) if err != nil { if mongo.IsDuplicateKeyError(err) { - return fmt.Errorf("%w: %s", store.ResourceAlreadyExistsError, err) + return fmt.Errorf("%w: %s", store.ErrResourceAlreadyExists, err) } return err } @@ -61,7 +61,7 @@ func (rc *ResourceRepository) Update(r *domain.Resource) error { err := singleResult.Err() if err != nil { if err == mongo.ErrNoDocuments { - return fmt.Errorf("%w: urn = %s", store.ResourceNotFoundError, r.Urn) + return fmt.Errorf("%w: urn = %s", store.ErrResourceNotFound, r.Urn) } return err } @@ -73,7 +73,7 @@ func (rc *ResourceRepository) GetByURN(urn string) (*domain.Resource, error) { err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) if err != nil { if err == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%w: %s", store.ResourceNotFoundError, err) + return nil, fmt.Errorf("%w: %s", store.ErrResourceNotFound, err) } return nil, err } @@ -100,7 +100,7 @@ func (rc *ResourceRepository) Delete(urn string) error { _, err := rc.collection.DeleteOne(context.TODO(), map[string]interface{}{"urn": urn}) if err != nil { if err == mongo.ErrNoDocuments { - return fmt.Errorf("%w: %s", store.ResourceNotFoundError, err) + return fmt.Errorf("%w: %s", store.ErrResourceNotFound, err) } return err } diff --git a/store/mongodb/resource_repository_test.go b/store/mongodb/resource_repository_test.go index 0d4094fc..ef1318e0 100644 --- a/store/mongodb/resource_repository_test.go +++ b/store/mongodb/resource_repository_test.go @@ -2,14 +2,15 @@ package mongodb import ( "errors" + "reflect" + "testing" + "time" + "github.com/odpf/entropy/domain" "github.com/odpf/entropy/store" "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/integration/mtest" - "reflect" - "testing" - "time" ) func TestNewResourceRepository(t *testing.T) { @@ -94,7 +95,7 @@ func TestResourceRepository_Create(t *testing.T) { UpdatedAt: time.Now(), }} }, - wantErr: store.ResourceAlreadyExistsError, + wantErr: store.ErrResourceAlreadyExists, }, } for _, tt := range tests { @@ -152,7 +153,7 @@ func TestResourceRepository_GetByURN(t *testing.T) { fields: func(mt *mtest.T) fields { return fields{mt.Coll} }, args: func(mt *mtest.T) args { return args{"p-testdata-gl-unknown-log"} }, want: func(mt *mtest.T) *domain.Resource { return nil }, - wantErr: store.ResourceNotFoundError, + wantErr: store.ErrResourceNotFound, }, } for _, tt := range tests { diff --git a/store/store.go b/store/store.go index 05ae9f5e..c263f5b2 100644 --- a/store/store.go +++ b/store/store.go @@ -11,10 +11,10 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ResourceAlreadyExistsError = errors.New("resource already exists") - ResourceNotFoundError = errors.New("no resource(s) found") - ModuleAlreadyExistsError = errors.New("module already exists") - ModuleNotFoundError = errors.New("no module(s) found") + ErrResourceAlreadyExists = errors.New("resource already exists") + ErrResourceNotFound = errors.New("no resource(s) found") + ErrModuleAlreadyExists = errors.New("module already exists") + ErrModuleNotFound = errors.New("no module(s) found") ) var ResourceRepositoryName = "resources" From 6376aaaeab5f1c54d99fa09820c2423b9b0e08d0 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 30 Mar 2022 11:25:18 +0530 Subject: [PATCH 07/20] feat: provider framework to pass creds to k8s/helm provider --- app/app.go | 45 ++++++++++-- domain/provider.go | 23 ++++++ go.mod | 2 +- modules/firehose/module.go | 100 ++++++++++++++++++++++++++- pkg/provider/helm/provider.go | 20 ++++-- pkg/provider/helm/release.go | 46 ++++++------ store/mongodb/provider_repository.go | 53 ++++++++++++++ store/store.go | 9 +++ 8 files changed, 262 insertions(+), 36 deletions(-) create mode 100644 domain/provider.go create mode 100644 store/mongodb/provider_repository.go diff --git a/app/app.go b/app/app.go index 6204a3fc..5ed15bac 100644 --- a/app/app.go +++ b/app/app.go @@ -3,16 +3,18 @@ package app import ( "context" "fmt" + "net/http" + "time" + grpc_recovery "github.com/grpc-ecosystem/go-grpc-middleware/recovery" "github.com/odpf/entropy/modules/firehose" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/pkg/module" + "github.com/odpf/entropy/pkg/provider/helm" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" "github.com/odpf/entropy/store/inmemory" "github.com/odpf/entropy/store/mongodb" - "net/http" - "time" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" grpc_zap "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap" @@ -33,6 +35,8 @@ import ( "google.golang.org/grpc" ) +var providerURN = "provider_name-kubernetes" + // Config contains the application configuration type Config struct { Service ServiceConfig `mapstructure:"service"` @@ -72,14 +76,31 @@ func RunServer(c *Config) error { mongoStore.Collection(store.ResourceRepositoryName), ) - moduleRepository := inmemory.NewModuleRepository() + providerRepository := mongodb.NewProviderRepository( + mongoStore.Collection(store.ProviderRepositoryName), + ) + moduleRepository := inmemory.NewModuleRepository() err = moduleRepository.Register(log.New(loggerInstance)) if err != nil { return err } - err = moduleRepository.Register(firehose.New()) + providerFetched, err := providerRepository.GetByURN(providerURN) + if err != nil { + return err + } + providerConfig := providerFetched.Configs + + kubeConfig := helm.ToKubeConfig(providerConfig) + + helmConfig := &helm.ProviderConfig{ + Kubernetes: kubeConfig, + } + + hp := helm.NewProvider(helmConfig) + + err = moduleRepository.Register(firehose.New(hp)) if err != nil { return err } @@ -160,5 +181,19 @@ func RunMigrations(c *Config) error { mongoStore.Collection(store.ResourceRepositoryName), ) - return resourceRepository.Migrate() + providerRepository := mongodb.NewProviderRepository( + mongoStore.Collection(store.ProviderRepositoryName), + ) + + err = resourceRepository.Migrate() + if err != nil { + return err + } + + err = providerRepository.Migrate() + if err != nil { + return err + } + + return nil } diff --git a/domain/provider.go b/domain/provider.go new file mode 100644 index 00000000..3febab12 --- /dev/null +++ b/domain/provider.go @@ -0,0 +1,23 @@ +package domain + +import ( + "strings" + "time" +) + +type Provider struct { + Urn string `bson:"urn"` + Name string `bson:"name"` + Kind string `bson:"kind"` + Configs map[string]interface{} `bson:"configs"` + Labels map[string]string `bson:"labels"` + CreatedAt time.Time `bson:"created_at"` + UpdatedAt time.Time `bson:"updated_at"` +} + +func GenerateProviderUrn(res *Provider) string { + return strings.Join([]string{ + sanitizeString(res.Name), + sanitizeString(res.Kind), + }, "-") +} diff --git a/go.mod b/go.mod index d549929f..db23b559 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.17 require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/mcuadros/go-defaults v1.2.0 + github.com/mitchellh/mapstructure v1.4.3 github.com/newrelic/go-agent/v3 v3.12.0 github.com/newrelic/go-agent/v3/integrations/nrgrpc v1.3.1 github.com/odpf/salt v0.0.0-20210929215807-5e1f68b4ec91 @@ -93,7 +94,6 @@ require ( github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.0 // indirect - github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/moby/locker v1.0.1 // indirect github.com/moby/spdystream v0.2.0 // indirect diff --git a/modules/firehose/module.go b/modules/firehose/module.go index a9627f74..14ec996d 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -3,12 +3,32 @@ package firehose import ( "errors" "fmt" + "reflect" "strings" + "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" + "github.com/odpf/entropy/pkg/provider/helm" gjs "github.com/xeipuuv/gojsonschema" ) +const ( + nameConfigString = "name" + repositoryConfigString = "repository" + chartConfigString = "chart" + versionConfigString = "version" + valuesConfigString = "values" + namespaceConfigString = "namespace" + timeoutConfigString = "timeout" + forceUpdateConfigString = "force_update" + recreatePodsConfigString = "recreate_pods" + waitConfigString = "wait" + waitForJobsConfigString = "wait_for_jobs" + replaceConfigString = "replace" + descriptionConfigString = "description" + CreateNamespaceConfigString = "create_namespace" +) + const configSchemaString = ` { "$schema": "http://json-schema.org/draft-07/schema#", @@ -229,13 +249,14 @@ const configSchemaString = ` type Module struct { schema *gjs.Schema + helm *helm.Provider } func (m *Module) ID() string { return "firehose" } -func New() *Module { +func New(helm *helm.Provider) *Module { schemaLoader := gjs.NewStringLoader(configSchemaString) schema, err := gjs.NewSchema(schemaLoader) if err != nil { @@ -243,15 +264,90 @@ func New() *Module { } return &Module{ schema: schema, + helm: helm, } } func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { + v := reflect.ValueOf(r.Configs[valuesConfigString]) + var values = make(map[string]interface{}) + if v.Kind() == reflect.Map { + for _, key := range v.MapKeys() { + strct := v.MapIndex(key) + values[key.String()] = strct.Interface() + } + } + + var rc helm.ReleaseConfig + if err := mapstructure.Decode(r.Configs, &rc); err != nil { + return domain.ResourceStatusError, err + } + + releaseConfig := helm.DefaultReleaseConfig() + releaseConfig.Values = values + + if r.Configs[nameConfigString] != nil { + releaseConfig.Name = r.Configs[nameConfigString].(string) + } + + if r.Configs[repositoryConfigString] != nil { + releaseConfig.Repository = r.Configs[repositoryConfigString].(string) + } + + if r.Configs[chartConfigString] != nil { + releaseConfig.Chart = r.Configs[chartConfigString].(string) + } + + if r.Configs[versionConfigString] != nil { + releaseConfig.Version = r.Configs[versionConfigString].(string) + } + + if r.Configs[namespaceConfigString] != nil { + releaseConfig.Namespace = r.Configs[namespaceConfigString].(string) + } + + if r.Configs[timeoutConfigString] != nil { + releaseConfig.Timeout = r.Configs[timeoutConfigString].(int) + } + + if r.Configs[descriptionConfigString] != nil { + releaseConfig.Description = r.Configs[descriptionConfigString].(string) + } + + if r.Configs[forceUpdateConfigString] != nil { + releaseConfig.ForceUpdate = r.Configs[forceUpdateConfigString].(bool) + } + + if r.Configs[recreatePodsConfigString] != nil { + releaseConfig.RecreatePods = r.Configs[recreatePodsConfigString].(bool) + } + + if r.Configs[waitConfigString] != nil { + releaseConfig.Wait = r.Configs[waitConfigString].(bool) + } + + if r.Configs[waitForJobsConfigString] != nil { + releaseConfig.WaitForJobs = r.Configs[waitForJobsConfigString].(bool) + } + + if r.Configs[replaceConfigString] != nil { + releaseConfig.Replace = r.Configs[replaceConfigString].(bool) + } + + if r.Configs[CreateNamespaceConfigString] != nil { + releaseConfig.CreateNamespace = r.Configs[CreateNamespaceConfigString].(bool) + } + + _, err := m.helm.Release(releaseConfig) + if err != nil { + return domain.ResourceStatusError, nil + } + return domain.ResourceStatusCompleted, nil } func (m *Module) Validate(r *domain.Resource) error { - resourceLoader := gjs.NewGoLoader(r.Configs) + resourceLoader := gjs.NewGoLoader(r.Configs["values"]) result, err := m.schema.Validate(resourceLoader) if err != nil { return fmt.Errorf("%w: %s", domain.ErrModuleConfigParseFailed, err) diff --git a/pkg/provider/helm/provider.go b/pkg/provider/helm/provider.go index a3ea42e0..43fb48bd 100644 --- a/pkg/provider/helm/provider.go +++ b/pkg/provider/helm/provider.go @@ -10,15 +10,15 @@ import ( "k8s.io/client-go/tools/clientcmd/api" ) -type providerConfig struct { +type ProviderConfig struct { // HelmDriver - The backend storage driver. Values are - configmap, secret, memory, sql HelmDriver string `default:"secret"` // Kubernetes configuration. Kubernetes KubernetesConfig } -func DefaultProviderConfig() *providerConfig { - defaultProviderConfig := new(providerConfig) +func DefaultProviderConfig() *ProviderConfig { + defaultProviderConfig := new(ProviderConfig) defaults.SetDefaults(defaultProviderConfig) return defaultProviderConfig } @@ -41,11 +41,11 @@ type KubernetesConfig struct { } type Provider struct { - config *providerConfig + config *ProviderConfig cliSettings *cli.EnvSettings } -func NewProvider(config *providerConfig) *Provider { +func NewProvider(config *ProviderConfig) *Provider { return &Provider{config: config, cliSettings: cli.New()} } @@ -80,3 +80,13 @@ func (p *Provider) getActionConfiguration(namespace string) (*action.Configurati } return actionConfig, nil } + +func ToKubeConfig(providerConfig map[string]interface{}) KubernetesConfig { + return KubernetesConfig{ + Host: providerConfig["host"].(string), + Insecure: providerConfig["insecure"].(bool), + ClientCertificate: providerConfig["clientCertificate"].(string), + ClientKey: providerConfig["clientKey"].(string), + ClusterCACertificate: providerConfig["clusterCACertificate"].(string), + } +} diff --git a/pkg/provider/helm/release.go b/pkg/provider/helm/release.go index 371c4db9..7b8fa8c0 100644 --- a/pkg/provider/helm/release.go +++ b/pkg/provider/helm/release.go @@ -18,45 +18,45 @@ import ( var ErrReleaseNotFound = errors.New("release not found") var ErrChartNotApplication = errors.New("helm chart is not an application chart") -type releaseConfig struct { +type ReleaseConfig struct { // Name - Release Name - Name string `valid:"required"` + Name string `json:"name" mapstructure:"name" valid:"required"` // Repository - Repository where to locate the requested chart. If is a URL the chart is installed without installing the repository. - Repository string `valid:"required"` + Repository string `json:"repository" mapstructure:"repository" valid:"required"` // Chart - Chart name to be installed. A path may be used. - Chart string `valid:"required"` + Chart string `json:"chart" mapstructure:"chart" valid:"required"` // Version - Specify the exact chart version to install. If this is not specified, the latest version is installed. - Version string + Version string `json:"version" mapstructure:"version"` // Values - Map of values in to pass to helm. - Values map[string]interface{} + Values map[string]interface{} `json:"values" mapstructure:"values"` // Namespace - Namespace to install the release into. - Namespace string `default:"default"` + Namespace string `json:"namespace" mapstructure:"namespace" default:"default"` // Timeout - Time in seconds to wait for any individual kubernetes operation. - Timeout int `default:"300"` + Timeout int `json:"timeout" mapstructure:"timeout" default:"300"` // ForceUpdate - Force resource update through delete/recreate if needed. - ForceUpdate bool `default:"false"` + ForceUpdate bool `json:"force_update" mapstructure:"force_update" default:"false"` // RecreatePods - Perform pods restart during upgrade/rollback - RecreatePods bool `default:"false"` + RecreatePods bool `json:"recreate_pods" mapstructure:"recreate_pods" default:"false"` // Wait - Will wait until all resources are in a ready state before marking the release as successful. - Wait bool `default:"true"` + Wait bool `json:"wait" mapstructure:"wait" default:"true"` // WaitForJobs - If wait is enabled, will wait until all Jobs have been completed before marking the release as successful. - WaitForJobs bool `default:"false"` + WaitForJobs bool `json:"wait_for_jobs" mapstructure:"wait_for_jobs" default:"false"` // Replace - Re-use the given name, even if that name is already used. This is unsafe in production - Replace bool `default:"false"` + Replace bool `json:"replace" mapstructure:"replace" default:"false"` // Description - Add a custom description - Description string + Description string `json:"description" mapstructure:"description"` // CreateNamespace - Create the namespace if it does not exist - CreateNamespace bool `default:"false"` + CreateNamespace bool `json:"create_namespace" mapstructure:"create_namespace" default:"false"` } -func DefaultReleaseConfig() *releaseConfig { - defaultReleaseConfig := new(releaseConfig) +func DefaultReleaseConfig() *ReleaseConfig { + defaultReleaseConfig := new(ReleaseConfig) defaults.SetDefaults(defaultReleaseConfig) return defaultReleaseConfig } type Release struct { - Config *releaseConfig + Config *ReleaseConfig Output ReleaseOutput } @@ -68,7 +68,7 @@ type ReleaseOutput struct { } // Release - creates or updates a helm release with its configs -func (p *Provider) Release(config *releaseConfig) (*Release, error) { +func (p *Provider) Release(config *ReleaseConfig) (*Release, error) { releaseExists, _ := p.resourceReleaseExists(config.Name, config.Namespace) if releaseExists { return p.update(config) @@ -76,7 +76,7 @@ func (p *Provider) Release(config *releaseConfig) (*Release, error) { return p.create(config) } -func (p *Provider) create(config *releaseConfig) (*Release, error) { +func (p *Provider) create(config *ReleaseConfig) (*Release, error) { actionConfig, err := p.getActionConfiguration(config.Namespace) if err != nil { return nil, fmt.Errorf("error while getting action configuration : %w", err) @@ -155,7 +155,7 @@ func (p *Provider) create(config *releaseConfig) (*Release, error) { }, nil } -func (p *Provider) update(config *releaseConfig) (*Release, error) { +func (p *Provider) update(config *ReleaseConfig) (*Release, error) { var rel *release.Release var err error @@ -230,7 +230,7 @@ func (p *Provider) update(config *releaseConfig) (*Release, error) { }, nil } -func (p *Provider) chartPathOptions(config *releaseConfig) (*action.ChartPathOptions, string) { +func (p *Provider) chartPathOptions(config *ReleaseConfig) (*action.ChartPathOptions, string) { repositoryURL, chartName := resolveChartName(config.Repository, strings.TrimSpace(config.Chart)) version := getVersion(config.Version) @@ -261,7 +261,7 @@ func getVersion(version string) string { return strings.TrimSpace(version) } -func (p *Provider) getChart(config *releaseConfig, name string, cpo *action.ChartPathOptions) (*chart.Chart, string, error) { +func (p *Provider) getChart(config *ReleaseConfig, name string, cpo *action.ChartPathOptions) (*chart.Chart, string, error) { // TODO: Add a lock as Load function blows up if accessed concurrently path, err := cpo.LocateChart(name, p.cliSettings) diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go new file mode 100644 index 00000000..fc48e80d --- /dev/null +++ b/store/mongodb/provider_repository.go @@ -0,0 +1,53 @@ +package mongodb + +import ( + "context" + "fmt" + "time" + + "github.com/odpf/entropy/store" + + "github.com/odpf/entropy/domain" + "go.mongodb.org/mongo-driver/mongo" +) + +type ProviderRepository struct { + collection *mongo.Collection +} + +func NewProviderRepository(collection *mongo.Collection) *ProviderRepository { + return &ProviderRepository{ + collection: collection, + } +} + +func (rc *ProviderRepository) Migrate() error { + return createUniqueIndex(rc.collection, "urn", 1) +} + +func (rc *ProviderRepository) Create(Provider *domain.Provider) error { + Provider.Urn = domain.GenerateProviderUrn(Provider) + Provider.CreatedAt = time.Now() + Provider.UpdatedAt = time.Now() + + _, err := rc.collection.InsertOne(context.TODO(), Provider) + if err != nil { + if mongo.IsDuplicateKeyError(err) { + return fmt.Errorf("%w: %s", store.ProviderAlreadyExistsError, err) + } + return err + } + return nil +} + +func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { + res := &domain.Provider{} + err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) + if err != nil { + if err == mongo.ErrNoDocuments { + return nil, fmt.Errorf("%w: %s", store.ProviderNotFoundError, err) + } + return nil, err + } + return res, nil +} diff --git a/store/store.go b/store/store.go index c263f5b2..c3dac3cc 100644 --- a/store/store.go +++ b/store/store.go @@ -15,9 +15,12 @@ var ( ErrResourceNotFound = errors.New("no resource(s) found") ErrModuleAlreadyExists = errors.New("module already exists") ErrModuleNotFound = errors.New("no module(s) found") + ErrProviderAlreadyExists = errors.New("provider already exists") + ErrProviderNotFound = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" +var ProviderRepositoryName = "providers" type ResourceRepository interface { Create(r *domain.Resource) error @@ -28,6 +31,12 @@ type ResourceRepository interface { Delete(urn string) error } +type ProviderRepository interface { + Create(r *domain.Resource) error + GetByURN(urn string) (*domain.Resource, error) + Migrate() error +} + type ModuleRepository interface { Register(module domain.Module) error Get(id string) (domain.Module, error) From 9a1c26e86641d40aa478afad708def9caccbd9f7 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 30 Mar 2022 11:39:23 +0530 Subject: [PATCH 08/20] chore: err var name restored --- store/store.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/store/store.go b/store/store.go index c3dac3cc..ab072396 100644 --- a/store/store.go +++ b/store/store.go @@ -11,12 +11,12 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ErrResourceAlreadyExists = errors.New("resource already exists") - ErrResourceNotFound = errors.New("no resource(s) found") - ErrModuleAlreadyExists = errors.New("module already exists") - ErrModuleNotFound = errors.New("no module(s) found") - ErrProviderAlreadyExists = errors.New("provider already exists") - ErrProviderNotFound = errors.New("no provider(s) found") + ResourceAlreadyExistsError = errors.New("resource already exists") + ResourceNotFoundError = errors.New("no resource(s) found") + ModuleAlreadyExistsError = errors.New("module already exists") + ModuleNotFoundError = errors.New("no module(s) found") + ProviderAlreadyExistsError = errors.New("provider already exists") + ProviderNotFoundError = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" From f77b96a83fc149640c9af72f48cdae1f892288f1 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 31 Mar 2022 22:21:29 +0530 Subject: [PATCH 09/20] feat: pass providers in resource payload --- app/app.go | 20 ++------------------ domain/provider.go | 3 ++- domain/resource.go | 1 + modules/firehose/module.go | 23 +++++++++++++++++------ store/mongodb/provider_repository.go | 4 ++-- 5 files changed, 24 insertions(+), 27 deletions(-) diff --git a/app/app.go b/app/app.go index 5ed15bac..3864d2d6 100644 --- a/app/app.go +++ b/app/app.go @@ -10,7 +10,6 @@ import ( "github.com/odpf/entropy/modules/firehose" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/pkg/module" - "github.com/odpf/entropy/pkg/provider/helm" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" "github.com/odpf/entropy/store/inmemory" @@ -35,8 +34,6 @@ import ( "google.golang.org/grpc" ) -var providerURN = "provider_name-kubernetes" - // Config contains the application configuration type Config struct { Service ServiceConfig `mapstructure:"service"` @@ -81,26 +78,13 @@ func RunServer(c *Config) error { ) moduleRepository := inmemory.NewModuleRepository() - err = moduleRepository.Register(log.New(loggerInstance)) - if err != nil { - return err - } - providerFetched, err := providerRepository.GetByURN(providerURN) + err = moduleRepository.Register(log.New(loggerInstance)) if err != nil { return err } - providerConfig := providerFetched.Configs - - kubeConfig := helm.ToKubeConfig(providerConfig) - - helmConfig := &helm.ProviderConfig{ - Kubernetes: kubeConfig, - } - - hp := helm.NewProvider(helmConfig) - err = moduleRepository.Register(firehose.New(hp)) + err = moduleRepository.Register(firehose.New(providerRepository)) if err != nil { return err } diff --git a/domain/provider.go b/domain/provider.go index 3febab12..9b1d30bd 100644 --- a/domain/provider.go +++ b/domain/provider.go @@ -9,6 +9,7 @@ type Provider struct { Urn string `bson:"urn"` Name string `bson:"name"` Kind string `bson:"kind"` + Parent string `bson:"parent"` Configs map[string]interface{} `bson:"configs"` Labels map[string]string `bson:"labels"` CreatedAt time.Time `bson:"created_at"` @@ -17,7 +18,7 @@ type Provider struct { func GenerateProviderUrn(res *Provider) string { return strings.Join([]string{ + sanitizeString(res.Parent), sanitizeString(res.Name), - sanitizeString(res.Kind), }, "-") } diff --git a/domain/resource.go b/domain/resource.go index 09356635..b7609ffb 100644 --- a/domain/resource.go +++ b/domain/resource.go @@ -23,6 +23,7 @@ type Resource struct { Kind string `bson:"kind"` Configs map[string]interface{} `bson:"configs"` Labels map[string]string `bson:"labels"` + Providers map[string]Provider `bson:"providers"` Status ResourceStatus `bson:"status"` CreatedAt time.Time `bson:"created_at"` UpdatedAt time.Time `bson:"updated_at"` diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 14ec996d..1137ba74 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -9,6 +9,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" "github.com/odpf/entropy/pkg/provider/helm" + "github.com/odpf/entropy/store/mongodb" gjs "github.com/xeipuuv/gojsonschema" ) @@ -248,27 +249,37 @@ const configSchemaString = ` ` type Module struct { - schema *gjs.Schema - helm *helm.Provider + schema *gjs.Schema + providerRepository *mongodb.ProviderRepository } func (m *Module) ID() string { return "firehose" } -func New(helm *helm.Provider) *Module { +func New(providerRepository *mongodb.ProviderRepository) *Module { schemaLoader := gjs.NewStringLoader(configSchemaString) schema, err := gjs.NewSchema(schemaLoader) if err != nil { return nil } return &Module{ - schema: schema, - helm: helm, + schema: schema, + providerRepository: providerRepository, } } func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { + kubeProviderConfig, err := m.providerRepository.GetConfigByURN(r.Providers["kubernetes"].Urn) + if err != nil { + return domain.ResourceStatusError, err + } + kubeConfig := helm.ToKubeConfig(kubeProviderConfig) + helmConfig := &helm.ProviderConfig{ + Kubernetes: kubeConfig, + } + helmProvider := helm.NewProvider(helmConfig) + v := reflect.ValueOf(r.Configs[valuesConfigString]) var values = make(map[string]interface{}) if v.Kind() == reflect.Map { @@ -338,7 +349,7 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { releaseConfig.CreateNamespace = r.Configs[CreateNamespaceConfigString].(bool) } - _, err := m.helm.Release(releaseConfig) + _, err = helmProvider.Release(releaseConfig) if err != nil { return domain.ResourceStatusError, nil } diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go index fc48e80d..b3a04b28 100644 --- a/store/mongodb/provider_repository.go +++ b/store/mongodb/provider_repository.go @@ -40,7 +40,7 @@ func (rc *ProviderRepository) Create(Provider *domain.Provider) error { return nil } -func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { +func (rc *ProviderRepository) GetConfigByURN(urn string) (map[string]interface{}, error) { res := &domain.Provider{} err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) if err != nil { @@ -49,5 +49,5 @@ func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { } return nil, err } - return res, nil + return res.Configs, nil } From ee8518d3a9c61fb142ba373751c004d183cb2be4 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 4 Apr 2022 20:05:50 +0530 Subject: [PATCH 10/20] feat: add provider service --- api/handlers/v1/{resource.go => server.go} | 127 +++++++++++- .../v1/{resource_test.go => server_test.go} | 30 ++- app/app.go | 14 +- domain/provider.go | 6 +- domain/resource.go | 7 +- go.mod | 2 +- go.sum | 2 + mocks/provider_repository.go | 186 ++++++++++++++++++ mocks/provider_service.go | 118 +++++++++++ modules/firehose/module.go | 107 +++------- pkg/provider/service.go | 46 +++++ .../providers}/helm/kube_rest.go | 0 .../providers}/helm/provider.go | 0 .../providers}/helm/release.go | 0 .../providers}/helm/release_test.go | 0 .../providers}/helm/status.go | 0 store/mongodb/provider_repository.go | 24 ++- store/store.go | 5 +- 18 files changed, 567 insertions(+), 107 deletions(-) rename api/handlers/v1/{resource.go => server.go} (66%) rename api/handlers/v1/{resource_test.go => server_test.go} (95%) create mode 100644 mocks/provider_repository.go create mode 100644 mocks/provider_service.go create mode 100644 pkg/provider/service.go rename {pkg/provider => plugins/providers}/helm/kube_rest.go (100%) rename {pkg/provider => plugins/providers}/helm/provider.go (100%) rename {pkg/provider => plugins/providers}/helm/release.go (100%) rename {pkg/provider => plugins/providers}/helm/release_test.go (100%) rename {pkg/provider => plugins/providers}/helm/status.go (100%) diff --git a/api/handlers/v1/resource.go b/api/handlers/v1/server.go similarity index 66% rename from api/handlers/v1/resource.go rename to api/handlers/v1/server.go index b70240af..eebdddc0 100644 --- a/api/handlers/v1/resource.go +++ b/api/handlers/v1/server.go @@ -3,9 +3,12 @@ package handlersv1 import ( "context" "errors" + "fmt" + "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" "github.com/odpf/entropy/pkg/module" + "github.com/odpf/entropy/pkg/provider" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" entropyv1beta1 "go.buf.build/odpf/gwv/odpf/proton/odpf/entropy/v1beta1" @@ -19,14 +22,17 @@ var ErrInternal = status.Error(codes.Internal, "internal server error") type APIServer struct { entropyv1beta1.UnimplementedResourceServiceServer + entropyv1beta1.UnimplementedProviderServiceServer resourceService resource.ServiceInterface moduleService module.ServiceInterface + providerService provider.ServiceInterface } -func NewApiServer(resourceService resource.ServiceInterface, moduleService module.ServiceInterface) *APIServer { +func NewApiServer(resourceService resource.ServiceInterface, moduleService module.ServiceInterface, providerService provider.ServiceInterface) *APIServer { return &APIServer{ resourceService: resourceService, moduleService: moduleService, + providerService: providerService, } } @@ -178,6 +184,51 @@ func (server APIServer) ApplyAction(ctx context.Context, request *entropyv1beta1 return response, nil } +func (server APIServer) CreateProvider(ctx context.Context, request *entropyv1beta1.CreateProviderRequest) (*entropyv1beta1.CreateProviderResponse, error) { + pro := providerFromProto(request.Provider) + pro.Urn = domain.GenerateProviderUrn(pro) + // TODO: add provider validation + + createdProvider, err := server.providerService.CreateProvider(ctx, pro) + if err != nil { + if errors.Is(err, store.ProviderAlreadyExistsError) { + return nil, status.Error(codes.AlreadyExists, "provider already exists") + } + return nil, ErrInternal + } + + responseProvider, err := providerToProto(createdProvider) + if err != nil { + return nil, ErrInternal + } + response := entropyv1beta1.CreateProviderResponse{ + Provider: responseProvider, + } + return &response, nil +} + +func (server APIServer) ListProviders(ctx context.Context, request *entropyv1beta1.ListProvidersRequest) (*entropyv1beta1.ListProvidersResponse, error) { + var responseProviders []*entropyv1beta1.Provider + providers, err := server.providerService.ListProviders(ctx, request.GetParent(), request.GetKind()) + if err != nil { + return nil, ErrInternal + } + for _, pro := range providers { + responseProvider, err := providerToProto(pro) + if err != nil { + return nil, ErrInternal + } + responseProviders = append(responseProviders, responseProvider) + } + if err != nil { + return nil, ErrInternal + } + response := entropyv1beta1.ListProvidersResponse{ + Providers: responseProviders, + } + return &response, nil +} + func (server APIServer) syncResource(ctx context.Context, updatedResource *domain.Resource) (*domain.Resource, error) { syncedResource, err := server.moduleService.Sync(ctx, updatedResource) if err != nil { @@ -216,12 +267,43 @@ func resourceToProto(res *domain.Resource) (*entropyv1beta1.Resource, error) { Kind: res.Kind, Configs: conf, Labels: res.Labels, + Providers: resourceProvidersToProto(res.Providers), Status: resourceStatusToProto(string(res.Status)), CreatedAt: timestamppb.New(res.CreatedAt), UpdatedAt: timestamppb.New(res.UpdatedAt), }, nil } +func resourceProvidersToProto(ps []domain.ProviderSelector) []*entropyv1beta1.ProviderSelector { + var providerSelectors []*entropyv1beta1.ProviderSelector + + for _, p := range ps { + selector := &entropyv1beta1.ProviderSelector{ + Urn: p.Urn, + Target: p.Target, + } + providerSelectors = append(providerSelectors, selector) + } + return providerSelectors +} + +func providerToProto(pro *domain.Provider) (*entropyv1beta1.Provider, error) { + conf, err := structpb.NewValue(pro.Configs) + if err != nil { + return nil, err + } + return &entropyv1beta1.Provider{ + Urn: pro.Urn, + Name: pro.Name, + Parent: pro.Parent, + Kind: pro.Kind, + Configs: conf, + Labels: pro.Labels, + CreatedAt: timestamppb.New(pro.CreatedAt), + UpdatedAt: timestamppb.New(pro.UpdatedAt), + }, nil +} + func resourceStatusToProto(status string) entropyv1beta1.Resource_Status { if resourceStatus, ok := entropyv1beta1.Resource_Status_value[status]; ok { return entropyv1beta1.Resource_Status(resourceStatus) @@ -231,11 +313,42 @@ func resourceStatusToProto(status string) entropyv1beta1.Resource_Status { func resourceFromProto(res *entropyv1beta1.Resource) *domain.Resource { return &domain.Resource{ - Urn: res.GetUrn(), - Name: res.GetName(), - Parent: res.GetParent(), - Kind: res.GetKind(), - Configs: res.GetConfigs().GetStructValue().AsMap(), - Labels: res.GetLabels(), + Urn: res.GetUrn(), + Name: res.GetName(), + Parent: res.GetParent(), + Kind: res.GetKind(), + Configs: res.GetConfigs().GetStructValue().AsMap(), + Labels: res.GetLabels(), + Providers: providerSelectorFromProto(res.GetProviders()), + } +} + +func providerSelectorFromProto(ps []*entropyv1beta1.ProviderSelector) []domain.ProviderSelector { + var providerSelectors []domain.ProviderSelector + + for _, p := range ps { + selector := domain.ProviderSelector{ + Urn: p.GetUrn(), + Target: p.GetTarget(), + } + providerSelectors = append(providerSelectors, selector) + } + return providerSelectors +} + +func providerFromProto(pro *entropyv1beta1.Provider) *domain.Provider { + var conf map[string]interface{} + err := mapstructure.Decode(pro.GetConfigs(), &conf) + if err != nil { + return nil + } + fmt.Printf("pro.GetConfigs(): %v\n", pro.GetConfigs()) + return &domain.Provider{ + Urn: pro.GetUrn(), + Name: pro.GetName(), + Parent: pro.GetParent(), + Kind: pro.GetKind(), + Configs: conf, + Labels: pro.GetLabels(), } } diff --git a/api/handlers/v1/resource_test.go b/api/handlers/v1/server_test.go similarity index 95% rename from api/handlers/v1/resource_test.go rename to api/handlers/v1/server_test.go index 8ab81be4..6cfbebb3 100644 --- a/api/handlers/v1/resource_test.go +++ b/api/handlers/v1/server_test.go @@ -102,7 +102,8 @@ func TestAPIServer_CreateResource(t *testing.T) { UpdatedAt: createdAt, }, nil) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -141,7 +142,8 @@ func TestAPIServer_CreateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(nil) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -191,7 +193,8 @@ func TestAPIServer_CreateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ErrModuleNotFound) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -241,7 +244,8 @@ func TestAPIServer_CreateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ErrModuleConfigParseFailed) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.CreateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("CreateResource() error = %v, wantErr %v", err, wantErr) @@ -352,7 +356,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { UpdatedAt: updatedAt, }, nil) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -384,7 +389,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { moduleService := &mocks.ModuleService{} - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -423,7 +429,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(store.ErrModuleNotFound) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -462,7 +469,8 @@ func TestAPIServer_UpdateResource(t *testing.T) { moduleService := &mocks.ModuleService{} moduleService.EXPECT().Validate(mock.Anything, mock.Anything).Return(domain.ErrModuleConfigParseFailed) - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.UpdateResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("UpdateResource() error = %v, wantErr %v", err, wantErr) @@ -620,7 +628,8 @@ func TestAPIServer_DeleteResource(t *testing.T) { moduleService := &mocks.ModuleService{} - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.DeleteResource(ctx, request) if !errors.Is(err, wantErr) { t.Errorf("DeleteResource() error = %v, wantErr %v", err, wantErr) @@ -644,7 +653,8 @@ func TestAPIServer_DeleteResource(t *testing.T) { moduleService := &mocks.ModuleService{} - server := NewApiServer(resourceService, moduleService) + providerService := &mocks.ProviderService{} + server := NewApiServer(resourceService, moduleService, providerService) got, err := server.DeleteResource(ctx, request) if errors.Is(err, nil) { t.Errorf("DeleteResource() got nil error") diff --git a/app/app.go b/app/app.go index 3864d2d6..702c49b0 100644 --- a/app/app.go +++ b/app/app.go @@ -10,6 +10,7 @@ import ( "github.com/odpf/entropy/modules/firehose" "github.com/odpf/entropy/modules/log" "github.com/odpf/entropy/pkg/module" + "github.com/odpf/entropy/pkg/provider" "github.com/odpf/entropy/pkg/resource" "github.com/odpf/entropy/store" "github.com/odpf/entropy/store/inmemory" @@ -91,6 +92,7 @@ func RunServer(c *Config) error { resourceService := resource.NewService(resourceRepository) moduleService := module.NewService(moduleRepository) + providerService := provider.NewService(providerRepository) muxServer, err := server.NewMux(server.Config{ Port: c.Service.Port, @@ -121,6 +123,11 @@ func RunServer(c *Config) error { return err } + err = gw.RegisterHandler(ctx, entropyv1beta1.RegisterProviderServiceHandlerFromEndpoint) + if err != nil { + return err + } + muxServer.SetGateway("/api", gw) muxServer.RegisterService( @@ -129,7 +136,12 @@ func RunServer(c *Config) error { ) muxServer.RegisterService( &entropyv1beta1.ResourceService_ServiceDesc, - handlersv1.NewApiServer(resourceService, moduleService), + handlersv1.NewApiServer(resourceService, moduleService, providerService), + ) + + muxServer.RegisterService( + &entropyv1beta1.ProviderService_ServiceDesc, + handlersv1.NewApiServer(resourceService, moduleService, providerService), ) muxServer.RegisterHandler("/ping", http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { diff --git a/domain/provider.go b/domain/provider.go index 9b1d30bd..5159678e 100644 --- a/domain/provider.go +++ b/domain/provider.go @@ -16,9 +16,9 @@ type Provider struct { UpdatedAt time.Time `bson:"updated_at"` } -func GenerateProviderUrn(res *Provider) string { +func GenerateProviderUrn(pro *Provider) string { return strings.Join([]string{ - sanitizeString(res.Parent), - sanitizeString(res.Name), + sanitizeString(pro.Parent), + sanitizeString(pro.Name), }, "-") } diff --git a/domain/resource.go b/domain/resource.go index b7609ffb..9871d74a 100644 --- a/domain/resource.go +++ b/domain/resource.go @@ -16,6 +16,11 @@ const ( ResourceStatusCompleted ResourceStatus = "STATUS_COMPLETED" ) +type ProviderSelector struct { + Urn string `bson:"urn"` + Target string `bson:"target"` +} + type Resource struct { Urn string `bson:"urn"` Name string `bson:"name"` @@ -23,7 +28,7 @@ type Resource struct { Kind string `bson:"kind"` Configs map[string]interface{} `bson:"configs"` Labels map[string]string `bson:"labels"` - Providers map[string]Provider `bson:"providers"` + Providers []ProviderSelector `bson:"providers"` Status ResourceStatus `bson:"status"` CreatedAt time.Time `bson:"created_at"` UpdatedAt time.Time `bson:"updated_at"` diff --git a/go.mod b/go.mod index db23b559..17c99489 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xeipuuv/gojsonschema v1.2.0 go.buf.build/odpf/gw/odpf/proton v1.1.9 - go.buf.build/odpf/gwv/odpf/proton v1.1.65 + go.buf.build/odpf/gwv/odpf/proton v1.1.76 go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.19.0 google.golang.org/grpc v1.43.0 diff --git a/go.sum b/go.sum index 89c2b7bc..9dabe755 100644 --- a/go.sum +++ b/go.sum @@ -1164,6 +1164,8 @@ go.buf.build/odpf/gwv/envoyproxy/protoc-gen-validate v1.1.3/go.mod h1:2Tg6rYIoDh go.buf.build/odpf/gwv/grpc-ecosystem/grpc-gateway v1.1.37/go.mod h1:UrBCdmHgaY/pLapYUMOq01c1yuzwT8AEBTsgpmzq2zo= go.buf.build/odpf/gwv/odpf/proton v1.1.65 h1:A0jh+7kMfiGWoSXpIoMOpO1sDTETLEo09y2wY7e40ZA= go.buf.build/odpf/gwv/odpf/proton v1.1.65/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= +go.buf.build/odpf/gwv/odpf/proton v1.1.76 h1:F8WbVUSHwGI1wHOewbLVIcshEJSJ/3HU6VKmSRNqtL0= +go.buf.build/odpf/gwv/odpf/proton v1.1.76/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= diff --git a/mocks/provider_repository.go b/mocks/provider_repository.go new file mode 100644 index 00000000..ea79ec63 --- /dev/null +++ b/mocks/provider_repository.go @@ -0,0 +1,186 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + domain "github.com/odpf/entropy/domain" + mock "github.com/stretchr/testify/mock" +) + +// ProviderRepository is an autogenerated mock type for the ProviderRepository type +type ProviderRepository struct { + mock.Mock +} + +type ProviderRepository_Expecter struct { + mock *mock.Mock +} + +func (_m *ProviderRepository) EXPECT() *ProviderRepository_Expecter { + return &ProviderRepository_Expecter{mock: &_m.Mock} +} + +// Create provides a mock function with given fields: r +func (_m *ProviderRepository) Create(r *domain.Provider) error { + ret := _m.Called(r) + + var r0 error + if rf, ok := ret.Get(0).(func(*domain.Provider) error); ok { + r0 = rf(r) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ProviderRepository_Create_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Create' +type ProviderRepository_Create_Call struct { + *mock.Call +} + +// Create is a helper method to define mock.On call +// - r *domain.Provider +func (_e *ProviderRepository_Expecter) Create(r interface{}) *ProviderRepository_Create_Call { + return &ProviderRepository_Create_Call{Call: _e.mock.On("Create", r)} +} + +func (_c *ProviderRepository_Create_Call) Run(run func(r *domain.Provider)) *ProviderRepository_Create_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(*domain.Provider)) + }) + return _c +} + +func (_c *ProviderRepository_Create_Call) Return(_a0 error) *ProviderRepository_Create_Call { + _c.Call.Return(_a0) + return _c +} + +// GetByURN provides a mock function with given fields: urn +func (_m *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { + ret := _m.Called(urn) + + var r0 *domain.Provider + if rf, ok := ret.Get(0).(func(string) *domain.Provider); ok { + r0 = rf(urn) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(urn) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderRepository_GetByURN_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetByURN' +type ProviderRepository_GetByURN_Call struct { + *mock.Call +} + +// GetByURN is a helper method to define mock.On call +// - urn string +func (_e *ProviderRepository_Expecter) GetByURN(urn interface{}) *ProviderRepository_GetByURN_Call { + return &ProviderRepository_GetByURN_Call{Call: _e.mock.On("GetByURN", urn)} +} + +func (_c *ProviderRepository_GetByURN_Call) Run(run func(urn string)) *ProviderRepository_GetByURN_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(string)) + }) + return _c +} + +func (_c *ProviderRepository_GetByURN_Call) Return(_a0 *domain.Provider, _a1 error) *ProviderRepository_GetByURN_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// List provides a mock function with given fields: filter +func (_m *ProviderRepository) List(filter map[string]string) ([]*domain.Provider, error) { + ret := _m.Called(filter) + + var r0 []*domain.Provider + if rf, ok := ret.Get(0).(func(map[string]string) []*domain.Provider); ok { + r0 = rf(filter) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(map[string]string) error); ok { + r1 = rf(filter) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderRepository_List_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'List' +type ProviderRepository_List_Call struct { + *mock.Call +} + +// List is a helper method to define mock.On call +// - filter map[string]string +func (_e *ProviderRepository_Expecter) List(filter interface{}) *ProviderRepository_List_Call { + return &ProviderRepository_List_Call{Call: _e.mock.On("List", filter)} +} + +func (_c *ProviderRepository_List_Call) Run(run func(filter map[string]string)) *ProviderRepository_List_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(map[string]string)) + }) + return _c +} + +func (_c *ProviderRepository_List_Call) Return(_a0 []*domain.Provider, _a1 error) *ProviderRepository_List_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// Migrate provides a mock function with given fields: +func (_m *ProviderRepository) Migrate() error { + ret := _m.Called() + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ProviderRepository_Migrate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Migrate' +type ProviderRepository_Migrate_Call struct { + *mock.Call +} + +// Migrate is a helper method to define mock.On call +func (_e *ProviderRepository_Expecter) Migrate() *ProviderRepository_Migrate_Call { + return &ProviderRepository_Migrate_Call{Call: _e.mock.On("Migrate")} +} + +func (_c *ProviderRepository_Migrate_Call) Run(run func()) *ProviderRepository_Migrate_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ProviderRepository_Migrate_Call) Return(_a0 error) *ProviderRepository_Migrate_Call { + _c.Call.Return(_a0) + return _c +} diff --git a/mocks/provider_service.go b/mocks/provider_service.go new file mode 100644 index 00000000..dcfa87bc --- /dev/null +++ b/mocks/provider_service.go @@ -0,0 +1,118 @@ +// Code generated by mockery v2.10.0. DO NOT EDIT. + +package mocks + +import ( + context "context" + + domain "github.com/odpf/entropy/domain" + mock "github.com/stretchr/testify/mock" +) + +// ProviderService is an autogenerated mock type for the ServiceInterface type +type ProviderService struct { + mock.Mock +} + +type ProviderService_Expecter struct { + mock *mock.Mock +} + +func (_m *ProviderService) EXPECT() *ProviderService_Expecter { + return &ProviderService_Expecter{mock: &_m.Mock} +} + +// CreateProvider provides a mock function with given fields: ctx, res +func (_m *ProviderService) CreateProvider(ctx context.Context, res *domain.Provider) (*domain.Provider, error) { + ret := _m.Called(ctx, res) + + var r0 *domain.Provider + if rf, ok := ret.Get(0).(func(context.Context, *domain.Provider) *domain.Provider); ok { + r0 = rf(ctx, res) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, *domain.Provider) error); ok { + r1 = rf(ctx, res) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderService_CreateProvider_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateProvider' +type ProviderService_CreateProvider_Call struct { + *mock.Call +} + +// CreateProvider is a helper method to define mock.On call +// - ctx context.Context +// - res *domain.Provider +func (_e *ProviderService_Expecter) CreateProvider(ctx interface{}, res interface{}) *ProviderService_CreateProvider_Call { + return &ProviderService_CreateProvider_Call{Call: _e.mock.On("CreateProvider", ctx, res)} +} + +func (_c *ProviderService_CreateProvider_Call) Run(run func(ctx context.Context, res *domain.Provider)) *ProviderService_CreateProvider_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*domain.Provider)) + }) + return _c +} + +func (_c *ProviderService_CreateProvider_Call) Return(_a0 *domain.Provider, _a1 error) *ProviderService_CreateProvider_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +// ListProviders provides a mock function with given fields: ctx, parent, kind +func (_m *ProviderService) ListProviders(ctx context.Context, parent string, kind string) ([]*domain.Provider, error) { + ret := _m.Called(ctx, parent, kind) + + var r0 []*domain.Provider + if rf, ok := ret.Get(0).(func(context.Context, string, string) []*domain.Provider); ok { + r0 = rf(ctx, parent, kind) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*domain.Provider) + } + } + + var r1 error + if rf, ok := ret.Get(1).(func(context.Context, string, string) error); ok { + r1 = rf(ctx, parent, kind) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// ProviderService_ListProviders_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListProviders' +type ProviderService_ListProviders_Call struct { + *mock.Call +} + +// ListProviders is a helper method to define mock.On call +// - ctx context.Context +// - parent string +// - kind string +func (_e *ProviderService_Expecter) ListProviders(ctx interface{}, parent interface{}, kind interface{}) *ProviderService_ListProviders_Call { + return &ProviderService_ListProviders_Call{Call: _e.mock.On("ListProviders", ctx, parent, kind)} +} + +func (_c *ProviderService_ListProviders_Call) Run(run func(ctx context.Context, parent string, kind string)) *ProviderService_ListProviders_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(string), args[2].(string)) + }) + return _c +} + +func (_c *ProviderService_ListProviders_Call) Return(_a0 []*domain.Provider, _a1 error) *ProviderService_ListProviders_Call { + _c.Call.Return(_a0, _a1) + return _c +} diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 1137ba74..8f7cbc56 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -8,7 +8,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" - "github.com/odpf/entropy/pkg/provider/helm" + "github.com/odpf/entropy/plugins/providers/helm" "github.com/odpf/entropy/store/mongodb" gjs "github.com/xeipuuv/gojsonschema" ) @@ -28,6 +28,7 @@ const ( replaceConfigString = "replace" descriptionConfigString = "description" CreateNamespaceConfigString = "create_namespace" + KUBERNETES = "kubernetes" ) const configSchemaString = ` @@ -270,95 +271,45 @@ func New(providerRepository *mongodb.ProviderRepository) *Module { } func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { - kubeProviderConfig, err := m.providerRepository.GetConfigByURN(r.Providers["kubernetes"].Urn) - if err != nil { - return domain.ResourceStatusError, err - } - kubeConfig := helm.ToKubeConfig(kubeProviderConfig) - helmConfig := &helm.ProviderConfig{ - Kubernetes: kubeConfig, - } - helmProvider := helm.NewProvider(helmConfig) - - v := reflect.ValueOf(r.Configs[valuesConfigString]) - var values = make(map[string]interface{}) - if v.Kind() == reflect.Map { - for _, key := range v.MapKeys() { - strct := v.MapIndex(key) - values[key.String()] = strct.Interface() + for _, p := range r.Providers { + provider, err := m.providerRepository.GetByURN(p.Urn) + if err != nil { + return domain.ResourceStatusError, err } - } - - var rc helm.ReleaseConfig - if err := mapstructure.Decode(r.Configs, &rc); err != nil { - return domain.ResourceStatusError, err - } - - releaseConfig := helm.DefaultReleaseConfig() - releaseConfig.Values = values - - if r.Configs[nameConfigString] != nil { - releaseConfig.Name = r.Configs[nameConfigString].(string) - } - - if r.Configs[repositoryConfigString] != nil { - releaseConfig.Repository = r.Configs[repositoryConfigString].(string) - } - - if r.Configs[chartConfigString] != nil { - releaseConfig.Chart = r.Configs[chartConfigString].(string) - } - - if r.Configs[versionConfigString] != nil { - releaseConfig.Version = r.Configs[versionConfigString].(string) - } - - if r.Configs[namespaceConfigString] != nil { - releaseConfig.Namespace = r.Configs[namespaceConfigString].(string) - } - if r.Configs[timeoutConfigString] != nil { - releaseConfig.Timeout = r.Configs[timeoutConfigString].(int) - } - - if r.Configs[descriptionConfigString] != nil { - releaseConfig.Description = r.Configs[descriptionConfigString].(string) - } - - if r.Configs[forceUpdateConfigString] != nil { - releaseConfig.ForceUpdate = r.Configs[forceUpdateConfigString].(bool) - } - - if r.Configs[recreatePodsConfigString] != nil { - releaseConfig.RecreatePods = r.Configs[recreatePodsConfigString].(bool) - } - - if r.Configs[waitConfigString] != nil { - releaseConfig.Wait = r.Configs[waitConfigString].(bool) - } + if provider.Kind == KUBERNETES { + releaseConfig := helm.DefaultReleaseConfig() + var values = make(map[string]interface{}) - if r.Configs[waitForJobsConfigString] != nil { - releaseConfig.WaitForJobs = r.Configs[waitForJobsConfigString].(bool) - } + kubeConfig := helm.ToKubeConfig(provider.Configs) + helmConfig := &helm.ProviderConfig{ + Kubernetes: kubeConfig, + } + helmProvider := helm.NewProvider(helmConfig) - if r.Configs[replaceConfigString] != nil { - releaseConfig.Replace = r.Configs[replaceConfigString].(bool) - } + v := reflect.ValueOf(r.Configs[valuesConfigString]) + if err := mapstructure.Decode(v, &values); err != nil { + return domain.ResourceStatusError, err + } + releaseConfig.Values = values - if r.Configs[CreateNamespaceConfigString] != nil { - releaseConfig.CreateNamespace = r.Configs[CreateNamespaceConfigString].(bool) - } + err := mapstructure.Decode(r.Configs, &releaseConfig) + if err != nil { + return domain.ResourceStatusError, err + } - _, err = helmProvider.Release(releaseConfig) - if err != nil { - return domain.ResourceStatusError, nil + _, err = helmProvider.Release(releaseConfig) + if err != nil { + return domain.ResourceStatusError, nil + } + } } return domain.ResourceStatusCompleted, nil } func (m *Module) Validate(r *domain.Resource) error { - resourceLoader := gjs.NewGoLoader(r.Configs["values"]) + resourceLoader := gjs.NewGoLoader(r.Configs[valuesConfigString]) result, err := m.schema.Validate(resourceLoader) if err != nil { return fmt.Errorf("%w: %s", domain.ErrModuleConfigParseFailed, err) diff --git a/pkg/provider/service.go b/pkg/provider/service.go new file mode 100644 index 00000000..e327f2f2 --- /dev/null +++ b/pkg/provider/service.go @@ -0,0 +1,46 @@ +package provider + +import ( + "context" + + "github.com/odpf/entropy/domain" + "github.com/odpf/entropy/store" +) + +type ServiceInterface interface { + CreateProvider(ctx context.Context, res *domain.Provider) (*domain.Provider, error) + ListProviders(ctx context.Context, parent string, kind string) ([]*domain.Provider, error) +} + +type Service struct { + providerRepository store.ProviderRepository +} + +func NewService(repository store.ProviderRepository) *Service { + return &Service{ + providerRepository: repository, + } +} + +func (s *Service) CreateProvider(ctx context.Context, pro *domain.Provider) (*domain.Provider, error) { + err := s.providerRepository.Create(pro) + if err != nil { + return nil, err + } + createdProvider, err := s.providerRepository.GetByURN(pro.Urn) + if err != nil { + return nil, err + } + return createdProvider, nil +} + +func (s *Service) ListProviders(ctx context.Context, parent string, kind string) ([]*domain.Provider, error) { + filter := map[string]string{} + if kind != "" { + filter["kind"] = kind + } + if parent != "" { + filter["parent"] = parent + } + return s.providerRepository.List(filter) +} diff --git a/pkg/provider/helm/kube_rest.go b/plugins/providers/helm/kube_rest.go similarity index 100% rename from pkg/provider/helm/kube_rest.go rename to plugins/providers/helm/kube_rest.go diff --git a/pkg/provider/helm/provider.go b/plugins/providers/helm/provider.go similarity index 100% rename from pkg/provider/helm/provider.go rename to plugins/providers/helm/provider.go diff --git a/pkg/provider/helm/release.go b/plugins/providers/helm/release.go similarity index 100% rename from pkg/provider/helm/release.go rename to plugins/providers/helm/release.go diff --git a/pkg/provider/helm/release_test.go b/plugins/providers/helm/release_test.go similarity index 100% rename from pkg/provider/helm/release_test.go rename to plugins/providers/helm/release_test.go diff --git a/pkg/provider/helm/status.go b/plugins/providers/helm/status.go similarity index 100% rename from pkg/provider/helm/status.go rename to plugins/providers/helm/status.go diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go index b3a04b28..a73c9ee7 100644 --- a/store/mongodb/provider_repository.go +++ b/store/mongodb/provider_repository.go @@ -40,14 +40,30 @@ func (rc *ProviderRepository) Create(Provider *domain.Provider) error { return nil } -func (rc *ProviderRepository) GetConfigByURN(urn string) (map[string]interface{}, error) { - res := &domain.Provider{} - err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(res) +func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { + pro := &domain.Provider{} + err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(pro) if err != nil { if err == mongo.ErrNoDocuments { return nil, fmt.Errorf("%w: %s", store.ProviderNotFoundError, err) } return nil, err } - return res.Configs, nil + return pro, nil +} + +func (rc *ProviderRepository) List(filter map[string]string) ([]*domain.Provider, error) { + var pro []*domain.Provider + cur, err := rc.collection.Find(context.TODO(), filter) + if err != nil { + return nil, err + } + err = cur.All(context.TODO(), &pro) + if err != nil { + if err == mongo.ErrNoDocuments { + return pro, nil + } + return nil, err + } + return pro, nil } diff --git a/store/store.go b/store/store.go index ab072396..4860a559 100644 --- a/store/store.go +++ b/store/store.go @@ -32,8 +32,9 @@ type ResourceRepository interface { } type ProviderRepository interface { - Create(r *domain.Resource) error - GetByURN(urn string) (*domain.Resource, error) + Create(r *domain.Provider) error + GetByURN(urn string) (*domain.Provider, error) + List(filter map[string]string) ([]*domain.Provider, error) Migrate() error } From 096fff50f3bdf7e7e5acfdbb5c903aab7092f91a Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Mon, 4 Apr 2022 20:22:01 +0530 Subject: [PATCH 11/20] chore: remove unused constants --- go.sum | 2 -- modules/firehose/module.go | 17 ++--------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/go.sum b/go.sum index 9dabe755..770e44b8 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,6 @@ go.buf.build/odpf/gw/odpf/proton v1.1.9 h1:iEdRUVVc/HwOqB7WRXjhXjR2pza2gyUbQ74G2 go.buf.build/odpf/gw/odpf/proton v1.1.9/go.mod h1:I9E8CF7w/690vRNWqBU6qDcUbi3Pi2THdn1yycBVTDQ= go.buf.build/odpf/gwv/envoyproxy/protoc-gen-validate v1.1.3/go.mod h1:2Tg6rYIoDhpl39Zd2+WBOF9uG4XxAOs0bK2Z2/bwTOc= go.buf.build/odpf/gwv/grpc-ecosystem/grpc-gateway v1.1.37/go.mod h1:UrBCdmHgaY/pLapYUMOq01c1yuzwT8AEBTsgpmzq2zo= -go.buf.build/odpf/gwv/odpf/proton v1.1.65 h1:A0jh+7kMfiGWoSXpIoMOpO1sDTETLEo09y2wY7e40ZA= -go.buf.build/odpf/gwv/odpf/proton v1.1.65/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.buf.build/odpf/gwv/odpf/proton v1.1.76 h1:F8WbVUSHwGI1wHOewbLVIcshEJSJ/3HU6VKmSRNqtL0= go.buf.build/odpf/gwv/odpf/proton v1.1.76/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 8f7cbc56..a8563ff4 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -14,21 +14,8 @@ import ( ) const ( - nameConfigString = "name" - repositoryConfigString = "repository" - chartConfigString = "chart" - versionConfigString = "version" - valuesConfigString = "values" - namespaceConfigString = "namespace" - timeoutConfigString = "timeout" - forceUpdateConfigString = "force_update" - recreatePodsConfigString = "recreate_pods" - waitConfigString = "wait" - waitForJobsConfigString = "wait_for_jobs" - replaceConfigString = "replace" - descriptionConfigString = "description" - CreateNamespaceConfigString = "create_namespace" - KUBERNETES = "kubernetes" + valuesConfigString = "values" + KUBERNETES = "kubernetes" ) const configSchemaString = ` From 98258cf1abc4f82ab07d8b62d6c55fc4af489a02 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 5 Apr 2022 15:10:12 +0530 Subject: [PATCH 12/20] chore: resolve merge conflicts --- api/handlers/v1/server.go | 2 +- store/mongodb/provider_repository.go | 4 ++-- store/store.go | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/api/handlers/v1/server.go b/api/handlers/v1/server.go index eebdddc0..f52d95f6 100644 --- a/api/handlers/v1/server.go +++ b/api/handlers/v1/server.go @@ -191,7 +191,7 @@ func (server APIServer) CreateProvider(ctx context.Context, request *entropyv1be createdProvider, err := server.providerService.CreateProvider(ctx, pro) if err != nil { - if errors.Is(err, store.ProviderAlreadyExistsError) { + if errors.Is(err, store.ErrProviderAlreadyExists) { return nil, status.Error(codes.AlreadyExists, "provider already exists") } return nil, ErrInternal diff --git a/store/mongodb/provider_repository.go b/store/mongodb/provider_repository.go index a73c9ee7..e893fb77 100644 --- a/store/mongodb/provider_repository.go +++ b/store/mongodb/provider_repository.go @@ -33,7 +33,7 @@ func (rc *ProviderRepository) Create(Provider *domain.Provider) error { _, err := rc.collection.InsertOne(context.TODO(), Provider) if err != nil { if mongo.IsDuplicateKeyError(err) { - return fmt.Errorf("%w: %s", store.ProviderAlreadyExistsError, err) + return fmt.Errorf("%w: %s", store.ErrProviderAlreadyExists, err) } return err } @@ -45,7 +45,7 @@ func (rc *ProviderRepository) GetByURN(urn string) (*domain.Provider, error) { err := rc.collection.FindOne(context.TODO(), map[string]interface{}{"urn": urn}).Decode(pro) if err != nil { if err == mongo.ErrNoDocuments { - return nil, fmt.Errorf("%w: %s", store.ProviderNotFoundError, err) + return nil, fmt.Errorf("%w: %s", store.ErrProviderNotFound, err) } return nil, err } diff --git a/store/store.go b/store/store.go index 4860a559..174271b3 100644 --- a/store/store.go +++ b/store/store.go @@ -11,12 +11,12 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ResourceAlreadyExistsError = errors.New("resource already exists") - ResourceNotFoundError = errors.New("no resource(s) found") - ModuleAlreadyExistsError = errors.New("module already exists") - ModuleNotFoundError = errors.New("no module(s) found") - ProviderAlreadyExistsError = errors.New("provider already exists") - ProviderNotFoundError = errors.New("no provider(s) found") + ErrResourceAlreadyExists = errors.New("resource already exists") + ErrResourceNotFound = errors.New("no resource(s) found") + ErrModuleAlreadyExists = errors.New("module already exists") + ErrModuleNotFound = errors.New("no module(s) found") + ErrProviderAlreadyExists = errors.New("provider already exists") + ErrProviderNotFound = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" From ed442eca41d5e4a0b3262e52a860cfacb52c8075 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 30 Mar 2022 11:39:23 +0530 Subject: [PATCH 13/20] chore: err var name restored --- store/store.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/store/store.go b/store/store.go index 174271b3..4860a559 100644 --- a/store/store.go +++ b/store/store.go @@ -11,12 +11,12 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ErrResourceAlreadyExists = errors.New("resource already exists") - ErrResourceNotFound = errors.New("no resource(s) found") - ErrModuleAlreadyExists = errors.New("module already exists") - ErrModuleNotFound = errors.New("no module(s) found") - ErrProviderAlreadyExists = errors.New("provider already exists") - ErrProviderNotFound = errors.New("no provider(s) found") + ResourceAlreadyExistsError = errors.New("resource already exists") + ResourceNotFoundError = errors.New("no resource(s) found") + ModuleAlreadyExistsError = errors.New("module already exists") + ModuleNotFoundError = errors.New("no module(s) found") + ProviderAlreadyExistsError = errors.New("provider already exists") + ProviderNotFoundError = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" From cc825ad2a3380b61e7cc5b18a3a32b11da0c67c5 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 5 Apr 2022 15:14:58 +0530 Subject: [PATCH 14/20] chore: resolve merge conflicts --- store/store.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/store/store.go b/store/store.go index 4860a559..174271b3 100644 --- a/store/store.go +++ b/store/store.go @@ -11,12 +11,12 @@ import ( // Custom errors which can be used by multiple DB vendors var ( - ResourceAlreadyExistsError = errors.New("resource already exists") - ResourceNotFoundError = errors.New("no resource(s) found") - ModuleAlreadyExistsError = errors.New("module already exists") - ModuleNotFoundError = errors.New("no module(s) found") - ProviderAlreadyExistsError = errors.New("provider already exists") - ProviderNotFoundError = errors.New("no provider(s) found") + ErrResourceAlreadyExists = errors.New("resource already exists") + ErrResourceNotFound = errors.New("no resource(s) found") + ErrModuleAlreadyExists = errors.New("module already exists") + ErrModuleNotFound = errors.New("no module(s) found") + ErrProviderAlreadyExists = errors.New("provider already exists") + ErrProviderNotFound = errors.New("no provider(s) found") ) var ResourceRepositoryName = "resources" From 35c25920fa8eb96173c4cb089fa8e6609bd3b54a Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 5 Apr 2022 15:26:35 +0530 Subject: [PATCH 15/20] chore: proton version update --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 17c99489..fedc76d4 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/stretchr/testify v1.7.0 github.com/xeipuuv/gojsonschema v1.2.0 go.buf.build/odpf/gw/odpf/proton v1.1.9 - go.buf.build/odpf/gwv/odpf/proton v1.1.76 + go.buf.build/odpf/gwv/odpf/proton v1.1.77 go.mongodb.org/mongo-driver v1.5.1 go.uber.org/zap v1.19.0 google.golang.org/grpc v1.43.0 diff --git a/go.sum b/go.sum index 770e44b8..5b1be5c0 100644 --- a/go.sum +++ b/go.sum @@ -1162,8 +1162,8 @@ go.buf.build/odpf/gw/odpf/proton v1.1.9 h1:iEdRUVVc/HwOqB7WRXjhXjR2pza2gyUbQ74G2 go.buf.build/odpf/gw/odpf/proton v1.1.9/go.mod h1:I9E8CF7w/690vRNWqBU6qDcUbi3Pi2THdn1yycBVTDQ= go.buf.build/odpf/gwv/envoyproxy/protoc-gen-validate v1.1.3/go.mod h1:2Tg6rYIoDhpl39Zd2+WBOF9uG4XxAOs0bK2Z2/bwTOc= go.buf.build/odpf/gwv/grpc-ecosystem/grpc-gateway v1.1.37/go.mod h1:UrBCdmHgaY/pLapYUMOq01c1yuzwT8AEBTsgpmzq2zo= -go.buf.build/odpf/gwv/odpf/proton v1.1.76 h1:F8WbVUSHwGI1wHOewbLVIcshEJSJ/3HU6VKmSRNqtL0= -go.buf.build/odpf/gwv/odpf/proton v1.1.76/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= +go.buf.build/odpf/gwv/odpf/proton v1.1.77 h1:vp0KIoAYYq46VF1f1/7WWzeSPYjvmNqRBIq3cqMbCLA= +go.buf.build/odpf/gwv/odpf/proton v1.1.77/go.mod h1:3VlD6BkZ7cmhyEb+pOdyjier/UO5QNT5rfBIgHyWzLM= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= From c5d68eb48c838ac7cfd224c22f41c867833cf2be Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 5 Apr 2022 15:36:42 +0530 Subject: [PATCH 16/20] chore: remove log statements --- api/handlers/v1/server.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/handlers/v1/server.go b/api/handlers/v1/server.go index f52d95f6..8a9ef69b 100644 --- a/api/handlers/v1/server.go +++ b/api/handlers/v1/server.go @@ -3,7 +3,6 @@ package handlersv1 import ( "context" "errors" - "fmt" "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" @@ -342,7 +341,6 @@ func providerFromProto(pro *entropyv1beta1.Provider) *domain.Provider { if err != nil { return nil } - fmt.Printf("pro.GetConfigs(): %v\n", pro.GetConfigs()) return &domain.Provider{ Urn: pro.GetUrn(), Name: pro.GetName(), From c9ec39b5f9da187292b9479eebef293dbb9d3ce6 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Tue, 5 Apr 2022 16:25:55 +0530 Subject: [PATCH 17/20] refactor: improved map-struct-value conversion --- api/handlers/v1/server.go | 8 +------- modules/firehose/module.go | 8 -------- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/api/handlers/v1/server.go b/api/handlers/v1/server.go index 8a9ef69b..e67b2a52 100644 --- a/api/handlers/v1/server.go +++ b/api/handlers/v1/server.go @@ -4,7 +4,6 @@ import ( "context" "errors" - "github.com/mitchellh/mapstructure" "github.com/odpf/entropy/domain" "github.com/odpf/entropy/pkg/module" "github.com/odpf/entropy/pkg/provider" @@ -336,17 +335,12 @@ func providerSelectorFromProto(ps []*entropyv1beta1.ProviderSelector) []domain.P } func providerFromProto(pro *entropyv1beta1.Provider) *domain.Provider { - var conf map[string]interface{} - err := mapstructure.Decode(pro.GetConfigs(), &conf) - if err != nil { - return nil - } return &domain.Provider{ Urn: pro.GetUrn(), Name: pro.GetName(), Parent: pro.GetParent(), Kind: pro.GetKind(), - Configs: conf, + Configs: pro.GetConfigs().GetStructValue().AsMap(), Labels: pro.GetLabels(), } } diff --git a/modules/firehose/module.go b/modules/firehose/module.go index a8563ff4..9c9cb0b0 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -3,7 +3,6 @@ package firehose import ( "errors" "fmt" - "reflect" "strings" "github.com/mitchellh/mapstructure" @@ -266,7 +265,6 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { if provider.Kind == KUBERNETES { releaseConfig := helm.DefaultReleaseConfig() - var values = make(map[string]interface{}) kubeConfig := helm.ToKubeConfig(provider.Configs) helmConfig := &helm.ProviderConfig{ @@ -274,12 +272,6 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { } helmProvider := helm.NewProvider(helmConfig) - v := reflect.ValueOf(r.Configs[valuesConfigString]) - if err := mapstructure.Decode(v, &values); err != nil { - return domain.ResourceStatusError, err - } - releaseConfig.Values = values - err := mapstructure.Decode(r.Configs, &releaseConfig) if err != nil { return domain.ResourceStatusError, err From bf14a3cc25590f0fa69b63ab15cbb0c91f9e0dc2 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 6 Apr 2022 20:35:35 +0530 Subject: [PATCH 18/20] feat: validate full resource config --- modules/firehose/module.go | 459 +++++++++++++++++------------- plugins/providers/helm/release.go | 11 +- 2 files changed, 261 insertions(+), 209 deletions(-) diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 9c9cb0b0..80dc37d1 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -13,8 +13,8 @@ import ( ) const ( - valuesConfigString = "values" - KUBERNETES = "kubernetes" + releaseConfigString = "release_configs" + KUBERNETES = "kubernetes" ) const configSchemaString = ` @@ -23,215 +23,266 @@ const configSchemaString = ` "$id": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "image": { - "type": "string" - }, - "replicas": { - "type": "number" - }, - "namespace": { - "type": "string" - }, - "cluster": { - "type": "string" - }, - "sink_type": { - "type": "string", - "enum": [ - "LOG", - "HTTP" - ] - }, - "state": { - "type": "string", - "enum": [ - "RUNNING", - "STOPPED" - ] - }, - "stop_date": { - "type": "string", - "format": "date-time" - }, - "description": { - "type": "string" - } - }, - "allOf": [ - { - "if": { - "properties": { - "sink_type": { - "const": "LOG" - } + "release_configs": { + "type": "object", + "properties": { + "name": { + "type": "string" }, - "required": [ - "sink_type" - ] - }, - "then": { - "properties": { - "configuration": { - "type": "object", - "properties": { - "KAFKA_RECORD_PARSER_MODE": { - "type": "string" - }, - "SOURCE_KAFKA_BROKERS": { - "type": "string" - }, - "SOURCE_KAFKA_TOPIC": { - "type": "string" - }, - "SOURCE_KAFKA_CONSUMER_GROUP_ID": { - "type": "string" - }, - "INPUT_SCHEMA_PROTO_CLASS": { - "type": "string" - } - }, - "required": [ - "KAFKA_RECORD_PARSER_MODE", - "SOURCE_KAFKA_BROKERS", - "SOURCE_KAFKA_TOPIC", - "SOURCE_KAFKA_CONSUMER_GROUP_ID", - "INPUT_SCHEMA_PROTO_CLASS" - ] - } - } - } - }, - { - "if": { - "properties": { - "sink_type": { - "const": "HTTP" - } + "repository": { + "type": "string" }, - "required": [ - "sink_type" - ] - }, - "then": { - "properties": { - "configuration": { - "type": "object", - "properties": { - "SOURCE_KAFKA_BROKERS": { - "type": "string" - }, - "SOURCE_KAFKA_TOPIC": { - "type": "string" - }, - "SOURCE_KAFKA_CONSUMER_GROUP_ID": { - "type": "string" - }, - "INPUT_SCHEMA_PROTO_CLASS": { - "type": "string" - }, - "SINK_HTTP_RETRY_STATUS_CODE_RANGES": { - "type": "string" - }, - "SINK_HTTP_REQUEST_LOG_STATUS_CODE_RANGES": { - "type": "string" - }, - "SINK_HTTP_REQUEST_TIMEOUT_MS": { - "type": "number" - }, - "SINK_HTTP_REQUEST_METHOD": { - "type": "string", - "enum": [ - "put", - "post" - ] - }, - "SINK_HTTP_MAX_CONNECTIONS": { - "type": "number" - }, - "SINK_HTTP_SERVICE_URL": { - "type": "string" - }, - "SINK_HTTP_HEADERS": { - "type": "string" - }, - "SINK_HTTP_PARAMETER_SOURCE": { - "type": "string", - "enum": [ - "key", - "message", - "disabled" - ] - }, - "SINK_HTTP_DATA_FORMAT": { - "type": "string", - "enum": [ - "proto", - "json" + "chart": { + "type": "string" + }, + "version": { + "type": "string" + }, + "namespace": { + "type": "string" + }, + "timeout": { + "type": "number" + }, + "force_update": { + "type": "boolean" + }, + "recreate_pods": { + "type": "boolean" + }, + "wait": { + "type": "boolean" + }, + "wait_for_jobs": { + "type": "boolean" + }, + "replace": { + "type": "boolean" + }, + "description": { + "type": "string" + }, + "create_namespace": { + "type": "boolean" + }, + "state": { + "type": "string", + "enum": [ + "RUNNING", + "STOPPED" + ] + }, + "values": { + "type": "object", + "properties": { + "image": { + "type": "string" + }, + "replicas": { + "type": "number" + }, + "namespace": { + "type": "string" + }, + "cluster": { + "type": "string" + }, + "sink_type": { + "type": "string", + "enum": [ + "LOG", + "HTTP" + ] + }, + "stop_date": { + "type": "string", + "format": "date-time" + }, + "description": { + "type": "string" + } + }, + "allOf": [ + { + "if": { + "properties": { + "sink_type": { + "const": "LOG" + } + }, + "required": [ + "sink_type" ] }, - "SINK_HTTP_OAUTH2_ENABLE": { - "type": "boolean" - }, - "SINK_HTTP_OAUTH2_ACCESS_TOKEN_URL": { - "type": "string" - }, - "SINK_HTTP_OAUTH2_CLIENT_NAME": { - "type": "string" - }, - "SINK_HTTP_OAUTH2_CLIENT_SECRET": { - "type": "string" - }, - "SINK_HTTP_OAUTH2_SCOPE": { - "type": "string" - }, - "SINK_HTTP_JSON_BODY_TEMPLATE": { - "type": "string" - }, - "SINK_HTTP_PARAMETER_PLACEMENT": { - "type": "string", - "enum": [ - "query", - "header" + "then": { + "properties": { + "configuration": { + "type": "object", + "properties": { + "KAFKA_RECORD_PARSER_MODE": { + "type": "string" + }, + "SOURCE_KAFKA_BROKERS": { + "type": "string" + }, + "SOURCE_KAFKA_TOPIC": { + "type": "string" + }, + "SOURCE_KAFKA_CONSUMER_GROUP_ID": { + "type": "string" + }, + "INPUT_SCHEMA_PROTO_CLASS": { + "type": "string" + } + }, + "required": [ + "KAFKA_RECORD_PARSER_MODE", + "SOURCE_KAFKA_BROKERS", + "SOURCE_KAFKA_TOPIC", + "SOURCE_KAFKA_CONSUMER_GROUP_ID", + "INPUT_SCHEMA_PROTO_CLASS" + ] + } + } + } + }, + { + "if": { + "properties": { + "sink_type": { + "const": "HTTP" + } + }, + "required": [ + "sink_type" ] }, - "SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS": { - "type": "string" + "then": { + "properties": { + "configuration": { + "type": "object", + "properties": { + "SOURCE_KAFKA_BROKERS": { + "type": "string" + }, + "SOURCE_KAFKA_TOPIC": { + "type": "string" + }, + "SOURCE_KAFKA_CONSUMER_GROUP_ID": { + "type": "string" + }, + "INPUT_SCHEMA_PROTO_CLASS": { + "type": "string" + }, + "SINK_HTTP_RETRY_STATUS_CODE_RANGES": { + "type": "string" + }, + "SINK_HTTP_REQUEST_LOG_STATUS_CODE_RANGES": { + "type": "string" + }, + "SINK_HTTP_REQUEST_TIMEOUT_MS": { + "type": "number" + }, + "SINK_HTTP_REQUEST_METHOD": { + "type": "string", + "enum": [ + "put", + "post" + ] + }, + "SINK_HTTP_MAX_CONNECTIONS": { + "type": "number" + }, + "SINK_HTTP_SERVICE_URL": { + "type": "string" + }, + "SINK_HTTP_HEADERS": { + "type": "string" + }, + "SINK_HTTP_PARAMETER_SOURCE": { + "type": "string", + "enum": [ + "key", + "message", + "disabled" + ] + }, + "SINK_HTTP_DATA_FORMAT": { + "type": "string", + "enum": [ + "proto", + "json" + ] + }, + "SINK_HTTP_OAUTH2_ENABLE": { + "type": "boolean" + }, + "SINK_HTTP_OAUTH2_ACCESS_TOKEN_URL": { + "type": "string" + }, + "SINK_HTTP_OAUTH2_CLIENT_NAME": { + "type": "string" + }, + "SINK_HTTP_OAUTH2_CLIENT_SECRET": { + "type": "string" + }, + "SINK_HTTP_OAUTH2_SCOPE": { + "type": "string" + }, + "SINK_HTTP_JSON_BODY_TEMPLATE": { + "type": "string" + }, + "SINK_HTTP_PARAMETER_PLACEMENT": { + "type": "string", + "enum": [ + "query", + "header" + ] + }, + "SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS": { + "type": "string" + } + }, + "required": [ + "SOURCE_KAFKA_BROKERS", + "SOURCE_KAFKA_TOPIC", + "SOURCE_KAFKA_CONSUMER_GROUP_ID", + "INPUT_SCHEMA_PROTO_CLASS", + "SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS", + "SINK_HTTP_PARAMETER_PLACEMENT", + "SINK_HTTP_JSON_BODY_TEMPLATE", + "SINK_HTTP_OAUTH2_SCOPE", + "SINK_HTTP_OAUTH2_CLIENT_SECRET", + "SINK_HTTP_OAUTH2_CLIENT_NAME", + "SINK_HTTP_OAUTH2_ACCESS_TOKEN_URL", + "SINK_HTTP_OAUTH2_ENABLE", + "SINK_HTTP_DATA_FORMAT", + "SINK_HTTP_PARAMETER_SOURCE", + "SINK_HTTP_HEADERS", + "SINK_HTTP_SERVICE_URL", + "SINK_HTTP_MAX_CONNECTIONS", + "SINK_HTTP_REQUEST_METHOD", + "SINK_HTTP_REQUEST_TIMEOUT_MS", + "SINK_HTTP_REQUEST_LOG_STATUS_CODE_RANGES", + "SINK_HTTP_RETRY_STATUS_CODE_RANGES" + ] + } + } } - }, - "required": [ - "SOURCE_KAFKA_BROKERS", - "SOURCE_KAFKA_TOPIC", - "SOURCE_KAFKA_CONSUMER_GROUP_ID", - "INPUT_SCHEMA_PROTO_CLASS", - "SINK_HTTP_PARAMETER_SCHEMA_PROTO_CLASS", - "SINK_HTTP_PARAMETER_PLACEMENT", - "SINK_HTTP_JSON_BODY_TEMPLATE", - "SINK_HTTP_OAUTH2_SCOPE", - "SINK_HTTP_OAUTH2_CLIENT_SECRET", - "SINK_HTTP_OAUTH2_CLIENT_NAME", - "SINK_HTTP_OAUTH2_ACCESS_TOKEN_URL", - "SINK_HTTP_OAUTH2_ENABLE", - "SINK_HTTP_DATA_FORMAT", - "SINK_HTTP_PARAMETER_SOURCE", - "SINK_HTTP_HEADERS", - "SINK_HTTP_SERVICE_URL", - "SINK_HTTP_MAX_CONNECTIONS", - "SINK_HTTP_REQUEST_METHOD", - "SINK_HTTP_REQUEST_TIMEOUT_MS", - "SINK_HTTP_REQUEST_LOG_STATUS_CODE_RANGES", - "SINK_HTTP_RETRY_STATUS_CODE_RANGES" - ] - } + } + ], + "required": [ + "image", + "replicas", + "namespace" + ] } - } + }, + "required": [ + "state" + ] } - ], - "required": [ - "image", - "replicas", - "namespace", - "state" - ] + } } ` @@ -272,7 +323,7 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { } helmProvider := helm.NewProvider(helmConfig) - err := mapstructure.Decode(r.Configs, &releaseConfig) + err := mapstructure.Decode(r.Configs[releaseConfigString], &releaseConfig) if err != nil { return domain.ResourceStatusError, err } @@ -288,7 +339,7 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { } func (m *Module) Validate(r *domain.Resource) error { - resourceLoader := gjs.NewGoLoader(r.Configs[valuesConfigString]) + resourceLoader := gjs.NewGoLoader(r.Configs) result, err := m.schema.Validate(resourceLoader) if err != nil { return fmt.Errorf("%w: %s", domain.ErrModuleConfigParseFailed, err) diff --git a/plugins/providers/helm/release.go b/plugins/providers/helm/release.go index 7b8fa8c0..41b60343 100644 --- a/plugins/providers/helm/release.go +++ b/plugins/providers/helm/release.go @@ -20,13 +20,13 @@ var ErrChartNotApplication = errors.New("helm chart is not an application chart" type ReleaseConfig struct { // Name - Release Name - Name string `json:"name" mapstructure:"name" valid:"required"` + Name string `json:"name" mapstructure:"name"` // Repository - Repository where to locate the requested chart. If is a URL the chart is installed without installing the repository. - Repository string `json:"repository" mapstructure:"repository" valid:"required"` + Repository string `json:"repository" mapstructure:"repository" default:"https://odpf.github.io/charts/"` // Chart - Chart name to be installed. A path may be used. - Chart string `json:"chart" mapstructure:"chart" valid:"required"` + Chart string `json:"chart" mapstructure:"chart" default:"firehose"` // Version - Specify the exact chart version to install. If this is not specified, the latest version is installed. - Version string `json:"version" mapstructure:"version"` + Version string `json:"version" mapstructure:"version" default:"0.1.1"` // Values - Map of values in to pass to helm. Values map[string]interface{} `json:"values" mapstructure:"values"` // Namespace - Namespace to install the release into. @@ -46,7 +46,8 @@ type ReleaseConfig struct { // Description - Add a custom description Description string `json:"description" mapstructure:"description"` // CreateNamespace - Create the namespace if it does not exist - CreateNamespace bool `json:"create_namespace" mapstructure:"create_namespace" default:"false"` + CreateNamespace bool `json:"create_namespace" mapstructure:"create_namespace" default:"false"` + State string `json:"state" mapstructure:"state"` } func DefaultReleaseConfig() *ReleaseConfig { From cbb88ebd8e9f2df49618a40530febe3d9fb4a811 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Wed, 6 Apr 2022 20:42:36 +0530 Subject: [PATCH 19/20] fix: remove unnecessary error block --- api/handlers/v1/server.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/handlers/v1/server.go b/api/handlers/v1/server.go index e67b2a52..567d36f5 100644 --- a/api/handlers/v1/server.go +++ b/api/handlers/v1/server.go @@ -211,6 +211,7 @@ func (server APIServer) ListProviders(ctx context.Context, request *entropyv1bet if err != nil { return nil, ErrInternal } + for _, pro := range providers { responseProvider, err := providerToProto(pro) if err != nil { @@ -218,9 +219,7 @@ func (server APIServer) ListProviders(ctx context.Context, request *entropyv1bet } responseProviders = append(responseProviders, responseProvider) } - if err != nil { - return nil, ErrInternal - } + response := entropyv1beta1.ListProvidersResponse{ Providers: responseProviders, } From 8103c3edc6fac536427c06651d0f550d66d8db34 Mon Sep 17 00:00:00 2001 From: Ishan Arya Date: Thu, 7 Apr 2022 15:16:27 +0530 Subject: [PATCH 20/20] fix: move firehose defaults to module --- modules/firehose/module.go | 20 ++++++++++++-------- plugins/providers/helm/release.go | 6 +++--- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/modules/firehose/module.go b/modules/firehose/module.go index 80dc37d1..27a9f2cf 100644 --- a/modules/firehose/module.go +++ b/modules/firehose/module.go @@ -13,8 +13,11 @@ import ( ) const ( - releaseConfigString = "release_configs" - KUBERNETES = "kubernetes" + releaseConfigString = "release_configs" + KUBERNETES = "kubernetes" + defaultRepositoryString = "https://odpf.github.io/charts/" + defaultChartString = "firehose" + defaultVersionString = "0.1.1" ) const configSchemaString = ` @@ -316,18 +319,19 @@ func (m *Module) Apply(r *domain.Resource) (domain.ResourceStatus, error) { if provider.Kind == KUBERNETES { releaseConfig := helm.DefaultReleaseConfig() + releaseConfig.Repository = defaultRepositoryString + releaseConfig.Chart = defaultChartString + releaseConfig.Version = defaultVersionString + err := mapstructure.Decode(r.Configs[releaseConfigString], &releaseConfig) + if err != nil { + return domain.ResourceStatusError, err + } kubeConfig := helm.ToKubeConfig(provider.Configs) helmConfig := &helm.ProviderConfig{ Kubernetes: kubeConfig, } helmProvider := helm.NewProvider(helmConfig) - - err := mapstructure.Decode(r.Configs[releaseConfigString], &releaseConfig) - if err != nil { - return domain.ResourceStatusError, err - } - _, err = helmProvider.Release(releaseConfig) if err != nil { return domain.ResourceStatusError, nil diff --git a/plugins/providers/helm/release.go b/plugins/providers/helm/release.go index 41b60343..e874db09 100644 --- a/plugins/providers/helm/release.go +++ b/plugins/providers/helm/release.go @@ -22,11 +22,11 @@ type ReleaseConfig struct { // Name - Release Name Name string `json:"name" mapstructure:"name"` // Repository - Repository where to locate the requested chart. If is a URL the chart is installed without installing the repository. - Repository string `json:"repository" mapstructure:"repository" default:"https://odpf.github.io/charts/"` + Repository string `json:"repository" mapstructure:"repository"` // Chart - Chart name to be installed. A path may be used. - Chart string `json:"chart" mapstructure:"chart" default:"firehose"` + Chart string `json:"chart" mapstructure:"chart"` // Version - Specify the exact chart version to install. If this is not specified, the latest version is installed. - Version string `json:"version" mapstructure:"version" default:"0.1.1"` + Version string `json:"version" mapstructure:"version"` // Values - Map of values in to pass to helm. Values map[string]interface{} `json:"values" mapstructure:"values"` // Namespace - Namespace to install the release into.