From 1228d2e9d25f13d7ca6a2327e8ca6370bb8063e7 Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Tue, 7 Dec 2021 09:34:45 +0530 Subject: [PATCH 1/4] feat: reading resources file --- cmd/serve_proxy.go | 18 ++++- config/config.go | 7 ++ resources_config/resources.yaml | 0 store/blob/resources_repository.go | 114 +++++++++++++++++++++++++++++ structs/resources.go | 8 ++ 5 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 resources_config/resources.yaml create mode 100644 store/blob/resources_repository.go create mode 100644 structs/resources.go diff --git a/cmd/serve_proxy.go b/cmd/serve_proxy.go index ba8b94b5f..6b95c8ad9 100644 --- a/cmd/serve_proxy.go +++ b/cmd/serve_proxy.go @@ -58,10 +58,24 @@ func proxyCommand(logger log.Logger, appConfig *config.Shield) *cli.Command { var cleanUpFunc []func() error var cleanUpProxies []func(ctx context.Context) error for _, service := range appConfig.Proxy.Services { + if service.ResourcesConfigPath == "" { + return errors.New("ruleset field cannot be left empty") + } + resoucesConfigFS, err := (&blobFactory{}).New(baseCtx, service.RulesPath, service.RulesPathSecret) + if err != nil { + return err + } + + resoucesRepo := blobstore.NewResourcesRepository(logger, resoucesConfigFS) + + if err := resoucesRepo.InitCache(baseCtx, ruleCacheRefreshDelay); err != nil { + return err + } + if service.RulesPath == "" { return errors.New("ruleset field cannot be left empty") } - blobFS, err := (&blobFactory{}).New(baseCtx, service.RulesPath, service.RulesPathSecret) + ruleFS, err := (&blobFactory{}).New(baseCtx, service.RulesPath, service.RulesPathSecret) if err != nil { return err } @@ -69,7 +83,7 @@ func proxyCommand(logger log.Logger, appConfig *config.Shield) *cli.Command { // TODO: option to use default http round tripper for http1.1 backends h2cProxy := proxy.NewH2c(proxy.NewH2cRoundTripper(logger), proxy.NewDirector()) - ruleRepo := blobstore.NewRuleRepository(logger, blobFS) + ruleRepo := blobstore.NewRuleRepository(logger, ruleFS) if err := ruleRepo.InitCache(baseCtx, ruleCacheRefreshDelay); err != nil { return err } diff --git a/config/config.go b/config/config.go index 2918dd562..94464c482 100644 --- a/config/config.go +++ b/config/config.go @@ -47,6 +47,13 @@ type Service struct { // Headers which will have user's email id IdentityProxyHeader string `yaml:"identity_proxy_header" mapstructure:"identity_proxy_header"` + + // ResourcesPath is a directory path where resources is defined + // that this service should implement + ResourcesConfigPath string `yaml:"resources_config_path" mapstructure:"resources_config_path"` + // ResourcesPathSecretSecret could be a env name, file path or actual value required + // to access ResourcesPathSecretPath files + ResourcesConfigPathSecret string `yaml:"resources_config_path_secret" mapstructure:"resources_config_path_secret"` } type NewRelic struct { diff --git a/resources_config/resources.yaml b/resources_config/resources.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/store/blob/resources_repository.go b/store/blob/resources_repository.go new file mode 100644 index 000000000..ee0cd9ff8 --- /dev/null +++ b/store/blob/resources_repository.go @@ -0,0 +1,114 @@ +package blob + +import ( + "context" + "io" + "strings" + "sync" + "time" + + "github.com/odpf/salt/log" + + "github.com/robfig/cron/v3" + + "github.com/ghodss/yaml" + "github.com/odpf/shield/store" + "github.com/odpf/shield/structs" + "github.com/pkg/errors" + "gocloud.dev/blob" +) + +type ResourcesRepository struct { + log log.Logger + mu *sync.Mutex + + cron *cron.Cron + bucket store.Bucket + cached []structs.Resources +} + +func (repo *ResourcesRepository) GetAll(ctx context.Context) ([]structs.Resources, error) { + repo.mu.Lock() + currentCache := repo.cached + repo.mu.Unlock() + if repo.cron != nil { + // cache must have been refreshed automatically, just return + return currentCache, nil + } + + err := repo.refresh(ctx) + return repo.cached, err +} + +func (repo *ResourcesRepository) refresh(ctx context.Context) error { + var resources []structs.Resources + + // get all items + it := repo.bucket.List(&blob.ListOptions{}) + for { + obj, err := it.Next(ctx) + if err != nil { + if err == io.EOF { + break + } + return err + } + + if obj.IsDir { + continue + } + if !(strings.HasSuffix(obj.Key, ".yaml") || strings.HasSuffix(obj.Key, ".yml")) { + continue + } + fileBytes, err := repo.bucket.ReadAll(ctx, obj.Key) + if err != nil { + return errors.Wrap(err, "bucket.ReadAll: "+obj.Key) + } + + var resource structs.Resources + if err := yaml.Unmarshal(fileBytes, &resource); err != nil { + return errors.Wrap(err, "yaml.Unmarshal: "+obj.Key) + } + if len(resource) == 0 { + continue + } + + resources = append(resources, resource) + } + + repo.mu.Lock() + repo.cached = resources + repo.mu.Unlock() + repo.log.Debug("rule cache refreshed", "ruleset_count", len(repo.cached)) + return nil +} + +func (repo *ResourcesRepository) InitCache(ctx context.Context, refreshDelay time.Duration) error { + repo.cron = cron.New(cron.WithChain( + cron.SkipIfStillRunning(cron.DefaultLogger), + )) + if _, err := repo.cron.AddFunc("@every "+refreshDelay.String(), func() { + if err := repo.refresh(ctx); err != nil { + repo.log.Warn("failed to refresh rule repository", "err", err) + } + }); err != nil { + return err + } + repo.cron.Start() + + // do it once right now + return repo.refresh(ctx) +} + +func (repo *ResourcesRepository) Close() error { + <-repo.cron.Stop().Done() + return repo.bucket.Close() +} + +func NewResourcesRepository(logger log.Logger, b store.Bucket) *ResourcesRepository { + return &ResourcesRepository{ + log: logger, + bucket: b, + mu: new(sync.Mutex), + } +} \ No newline at end of file diff --git a/structs/resources.go b/structs/resources.go new file mode 100644 index 000000000..6f6a736e5 --- /dev/null +++ b/structs/resources.go @@ -0,0 +1,8 @@ +package structs + +type Resources []Resource + +type Resource struct { + Name string `json:"name" yaml:"name"` + Actions map[string][]string `json:"actions" yaml:"actions"` +} From 69477c67d8d83e7279047d44af0e441caa8a833c Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Tue, 7 Dec 2021 09:50:47 +0530 Subject: [PATCH 2/4] chore: correct gitignore to exclude protos --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 188ef1018..723244948 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ proxies/*.yml .temp temp shield +!proto/* .shield.yaml dist coverage.txt From 0ea5ea5f8d4626b8202ec3f287aa9076c549918a Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Tue, 7 Dec 2021 13:40:59 +0530 Subject: [PATCH 3/4] fix: loading yaml for resources config --- .shield.sample.yaml | 3 ++- cmd/serve_proxy.go | 8 +++++++- resources_config/resources.yaml | 18 ++++++++++++++++++ store/blob/resources_repository.go | 28 +++++++++++++++++++++------- structs/resources.go | 2 -- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/.shield.sample.yaml b/.shield.sample.yaml index cce305824..fbf5a428d 100644 --- a/.shield.sample.yaml +++ b/.shield.sample.yaml @@ -26,4 +26,5 @@ proxy: # secret string "val://user:password" # # +optional - # ruleset_secret: env://TEST_RULESET_SECRET \ No newline at end of file + # ruleset_secret: env://TEST_RULESET_SECRET + resources_config_path: file://absolute_path_to_rules_directory \ No newline at end of file diff --git a/cmd/serve_proxy.go b/cmd/serve_proxy.go index 6b95c8ad9..81a83a878 100644 --- a/cmd/serve_proxy.go +++ b/cmd/serve_proxy.go @@ -61,7 +61,7 @@ func proxyCommand(logger log.Logger, appConfig *config.Shield) *cli.Command { if service.ResourcesConfigPath == "" { return errors.New("ruleset field cannot be left empty") } - resoucesConfigFS, err := (&blobFactory{}).New(baseCtx, service.RulesPath, service.RulesPathSecret) + resoucesConfigFS, err := (&blobFactory{}).New(baseCtx, service.ResourcesConfigPath, service.ResourcesConfigPathSecret) if err != nil { return err } @@ -72,6 +72,12 @@ func proxyCommand(logger log.Logger, appConfig *config.Shield) *cli.Command { return err } + // @TODO: Use this to get resources config + _, err = resoucesRepo.GetAll(baseCtx) + if err != nil { + return err + } + if service.RulesPath == "" { return errors.New("ruleset field cannot be left empty") } diff --git a/resources_config/resources.yaml b/resources_config/resources.yaml index e69de29bb..12668b9f1 100644 --- a/resources_config/resources.yaml +++ b/resources_config/resources.yaml @@ -0,0 +1,18 @@ +resources: + - name: dagger + actions: + read: + - team_admin + - dagger_manager + - org_admin + delete: + - team_admin + - org_admin + - name: firehose + actions: + read: + - team_admin + - firehose_manager + delete: + - team_admin + - org_admin \ No newline at end of file diff --git a/store/blob/resources_repository.go b/store/blob/resources_repository.go index ee0cd9ff8..2a0a31d20 100644 --- a/store/blob/resources_repository.go +++ b/store/blob/resources_repository.go @@ -18,16 +18,25 @@ import ( "gocloud.dev/blob" ) +type Resources struct { + Resources []Resource `json:"resources" yaml:"resources"` +} + +type Resource struct { + Name string `json:"name" yaml:"name"` + Actions map[string][]string `json:"actions" yaml:"actions"` +} + type ResourcesRepository struct { log log.Logger mu *sync.Mutex cron *cron.Cron bucket store.Bucket - cached []structs.Resources + cached []structs.Resource } -func (repo *ResourcesRepository) GetAll(ctx context.Context) ([]structs.Resources, error) { +func (repo *ResourcesRepository) GetAll(ctx context.Context) ([]structs.Resource, error) { repo.mu.Lock() currentCache := repo.cached repo.mu.Unlock() @@ -41,7 +50,7 @@ func (repo *ResourcesRepository) GetAll(ctx context.Context) ([]structs.Resource } func (repo *ResourcesRepository) refresh(ctx context.Context) error { - var resources []structs.Resources + var resources []structs.Resource // get all items it := repo.bucket.List(&blob.ListOptions{}) @@ -65,15 +74,20 @@ func (repo *ResourcesRepository) refresh(ctx context.Context) error { return errors.Wrap(err, "bucket.ReadAll: "+obj.Key) } - var resource structs.Resources + var resource Resources if err := yaml.Unmarshal(fileBytes, &resource); err != nil { return errors.Wrap(err, "yaml.Unmarshal: "+obj.Key) } - if len(resource) == 0 { + if len(resource.Resources) == 0 { continue } - resources = append(resources, resource) + for _, res := range resource.Resources { + resources = append(resources, structs.Resource{ + Name: res.Name, + Actions: res.Actions, + }) + } } repo.mu.Lock() @@ -111,4 +125,4 @@ func NewResourcesRepository(logger log.Logger, b store.Bucket) *ResourcesReposit bucket: b, mu: new(sync.Mutex), } -} \ No newline at end of file +} diff --git a/structs/resources.go b/structs/resources.go index 6f6a736e5..4b5a7efc4 100644 --- a/structs/resources.go +++ b/structs/resources.go @@ -1,7 +1,5 @@ package structs -type Resources []Resource - type Resource struct { Name string `json:"name" yaml:"name"` Actions map[string][]string `json:"actions" yaml:"actions"` From c02b5907988c28e94b1284c7e2a9ec3b0312c888 Mon Sep 17 00:00:00 2001 From: Kartik Verma Date: Tue, 7 Dec 2021 15:09:28 +0530 Subject: [PATCH 4/4] fix: correct logs --- store/blob/resources_repository.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/blob/resources_repository.go b/store/blob/resources_repository.go index 2a0a31d20..0f3600926 100644 --- a/store/blob/resources_repository.go +++ b/store/blob/resources_repository.go @@ -93,7 +93,7 @@ func (repo *ResourcesRepository) refresh(ctx context.Context) error { repo.mu.Lock() repo.cached = resources repo.mu.Unlock() - repo.log.Debug("rule cache refreshed", "ruleset_count", len(repo.cached)) + repo.log.Debug("resource config cache refreshed", "resource_config_count", len(repo.cached)) return nil } @@ -103,7 +103,7 @@ func (repo *ResourcesRepository) InitCache(ctx context.Context, refreshDelay tim )) if _, err := repo.cron.AddFunc("@every "+refreshDelay.String(), func() { if err := repo.refresh(ctx); err != nil { - repo.log.Warn("failed to refresh rule repository", "err", err) + repo.log.Warn("failed to refresh resource config repository", "err", err) } }); err != nil { return err