feat: implement the feature #313

Makefile

@@ -3,7 +3,7 @@ COMMIT := $(shell git rev-parse --short HEAD)
 TAG := "$(shell git rev-list --tags --max-count=1)"
 VERSION := "$(shell git describe --tags ${TAG})-next"
 BUILD_DIR=dist
-PROTON_COMMIT := "5416f4a08b424136c1ca342281cb0e41f9319f3e"
+PROTON_COMMIT := "4e3b6b5c5b51be27527a07d713e4caf076792afe"
 
 .PHONY: all build clean test tidy vet proto setup format generate
 

api/handler/v1beta1/adapter.go

@@ -76,6 +76,7 @@ func (a *adapter) FromProviderConfigProto(pc *guardianv1beta1.ProviderConfig) *d
 
 			resources = append(resources, &domain.ResourceConfig{
 				Type:   r.GetType(),
+				Filter: r.GetFilter(),
 				Policy: a.fromPolicyConfigProto(r.GetPolicy()),
 				Roles:  roles,
 			})

cli/job.go

@@ -100,9 +100,9 @@ func runJobCmd() *cobra.Command {
 			if job == nil {
 				return fmt.Errorf("invalid job name: %s", jobName)
 			}
-			if err := job(context.Background()); err != nil {
+			/*if err := job(context.Background()); err != nil {
 				return fmt.Errorf(`failed to run job "%s": %w`, jobName, err)
-			}
+			}*/
 
 			return nil
 		},

core/provider/service.go

@@ -2,7 +2,10 @@ package provider
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/odpf/guardian/pkg/evaluator"
+	"reflect"
 	"strings"
 	"time"
 
@@ -450,14 +453,42 @@ func (s *Service) getResources(ctx context.Context, p *domain.Provider) ([]*doma
 		return nil, err
 	}
 
+	resourceTypeFilterMap := make(map[string]string)
+	for _, rc := range p.Config.Resources {
+		if len(rc.Filter) > 0 {
+			resourceTypeFilterMap[rc.Type] = rc.Filter
+		}
+	}
 	res, err := c.GetResources(p.Config)
 	if err != nil {
 		return nil, fmt.Errorf("error fetching resources for %v: %w", p.ID, err)
 	}
 
+	filteredResources := make([]*domain.Resource, 0)
+	for _, r := range res {
+		if filterExpression, ok := resourceTypeFilterMap[r.Type]; ok {
+			resourceMap, err := structToMap(r)
+			if err != nil {
+				return nil, fmt.Errorf("parsing resource struct to map: %w", err)
+			}
+
+			v, err := evaluator.Expression(filterExpression).EvaluateWithVars(map[string]interface{}{
+				"resource": resourceMap,
+			})
+			if err != nil {
+				return nil, err
+			}
+			if !reflect.ValueOf(v).IsZero() {
+				filteredResources = append(filteredResources, r)
+			}
+		} else {
+			filteredResources = append(filteredResources, r)
+		}
+	}
+
 	for _, er := range existingResources {
 		isFound := false
-		for _, r := range res {
+		for _, r := range filteredResources {
 			if er.URN == r.URN {
 				existingMetadata := er.Details
 				if existingMetadata != nil {
@@ -482,7 +513,7 @@ func (s *Service) getResources(ctx context.Context, p *domain.Provider) ([]*doma
 			resources = append(resources, er)
 		}
 	}
-	for _, r := range res {
+	for _, r := range filteredResources {
 		isAdded := false
 		for _, rr := range resources {
 			if r.URN == rr.URN {
@@ -564,3 +595,20 @@ func validateDuration(d string) error {
 	}
 	return nil
 }
+
+func structToMap(item interface{}) (map[string]interface{}, error) {
+	result := map[string]interface{}{}
+
+	if item != nil {
+		jsonString, err := json.Marshal(item)
+		if err != nil {
+			return nil, err
+		}
+
+		if err := json.Unmarshal(jsonString, &result); err != nil {
+			return nil, err
+		}
+	}
+
+	return result, nil
+}

core/provider/service_test.go

@@ -312,6 +312,43 @@ func (s *ServiceTestSuite) TestFetchResources() {
 
 		s.Nil(actualError)
 	})
+
+	s.Run("should upsert filter resources on success", func() {
+		providersWithResourceFilter := []*domain.Provider{
+			{
+				ID:   "1",
+				Type: mockProviderType,
+				URN:  mockProvider,
+				Config: &domain.ProviderConfig{Resources: []*domain.ResourceConfig{
+					{Type: "dataset", Filter: "$resource.urn == 'resource2'"},
+				}},
+			},
+		}
+		s.mockProviderRepository.EXPECT().Find(mock.AnythingOfType("*context.emptyCtx")).Return(providersWithResourceFilter, nil).Once()
+		expectedResources := []*domain.Resource{}
+		for _, p := range providersWithResourceFilter {
+			resources := []*domain.Resource{
+				{
+					ProviderType: p.Type,
+					ProviderURN:  p.URN,
+					Type:         "dataset",
+					URN:          "resource1",
+				}, {
+					ProviderType: p.Type,
+					ProviderURN:  p.URN,
+					Type:         "dataset",
+					URN:          "resource2",
+				},
+			}
+			s.mockProvider.On("GetResources", p.Config).Return(resources, nil).Once()
+			expectedResources = append(expectedResources, resources[1])
+		}
+		s.mockResourceService.On("BulkUpsert", mock.Anything, expectedResources).Return(nil)
+		s.mockResourceService.On("Find", mock.Anything, mock.Anything).Return([]*domain.Resource{}, nil).Once()
+		actualError := s.service.FetchResources(context.Background())
+
+		s.Nil(actualError)
+	})
 }
 
 func (s *ServiceTestSuite) TestGrantAccess() {

docs/docs/providers/bigquery.md

@@ -71,6 +71,7 @@ resources:
         permissions:
           - WRITER
   - type: table
+    filter: $resource.name endsWith transtaction
     policy:
       id: my_policy
       version: 1

docs/docs/reference/provider.md

@@ -16,6 +16,7 @@ config:
     allow_active_access_extension_in: 24h
   resources:
     - type: "dataset"
+      filter: $resource.name == 'playground'
       policy:
         id: "my-policy"
         version: 1
@@ -62,11 +63,12 @@ updated_at: "2022-10-26T07:41:52.676004Z"
 
 ### `ResourceConfig`
 
-| Field | Type | Description | Required | 
-| :----- | :--------- | :---------- | :------ | 
-| `type`    | `string` | Possible values for the Resource Type:<br/> - BigQuery: [string(BigQuery)](../providers/bigquery.md#bigqueryresourcetype) <br/> - Google Cloud Storage: [string(GCS)](../providers/gcs#gcs-resource-types)<br/> - Metabase: [string(Metabase)](../providers/metabase.md#metabaseresourcetype) <br/> - Graffana: [string(Graffana)](../providers/grafana.md#grafanaresourcetype) <br/> - Tableau: [string(Tableau)](../providers/tableau.md#grafana-resource-type) <br/> - Google Cloud IAM: [string(GCloudIAM)](../providers/gcloud_iam.md#gcloudiamresourcetype) <br/> - No-Op: [string(No-Op)](../providers/noop)  | Yes|
-| `policy`  | `object(id: string, version: int)` | Approval policy config that want to be applied to this resource config. Example: `id: approval_policy_x, version: 1` |Yes|
-| `roles[]` | [`object(Role)`](provider.md#role) |List of resource permissions mapping|Yes|
+| Field     | Type | Description                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         | Required | 
+|:----------| :--------- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:---------| 
+| `type`    | `string` | Possible values for the Resource Type:<br/> - BigQuery: [string(BigQuery)](../providers/bigquery.md#bigqueryresourcetype) <br/> - Google Cloud Storage: [string(GCS)](../providers/gcs#gcs-resource-types)<br/> - Metabase: [string(Metabase)](../providers/metabase.md#metabaseresourcetype) <br/> - Graffana: [string(Graffana)](../providers/grafana.md#grafanaresourcetype) <br/> - Tableau: [string(Tableau)](../providers/tableau.md#grafana-resource-type) <br/> - Google Cloud IAM: [string(GCloudIAM)](../providers/gcloud_iam.md#gcloudiamresourcetype) <br/> - No-Op: [string(No-Op)](../providers/noop) | Yes      |
+| `filter`  | `string` | Filter condition to add a specific set of resources match with condition. Example: `filter: $resource.name endsWith audit`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | No       |
+| `policy`  | `object(id: string, version: int)` | Approval policy config that want to be applied to this resource config. Example: `id: approval_policy_x, version: 1`                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Yes      |
+| `roles[]` | [`object(Role)`](provider.md#role) | List of resource permissions mapping                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | Yes      |
 
 ### `Role`
 

domain/provider.go

@@ -40,6 +40,7 @@ type PolicyConfig struct {
 // ResourceConfig is the configuration for a resource type within a provider
 type ResourceConfig struct {
 	Type   string        `json:"type" yaml:"type" validate:"required"`
+	Filter string        `json:"filter" yaml:"type"`
 	Policy *PolicyConfig `json:"policy" yaml:"policy"`
 	Roles  []*Role       `json:"roles" yaml:"roles" validate:"required"`
 }