enhancement: Add filtering in the ListPolicies RPC (#1642)

* enhancement: Add new FilterPolicies RPC Signed-off-by: Sam Lock <sam@swlock.co.uk> * Adds support to filter by `name` (regex), `scope` and `version`. * Adds application defined function for SQLite driver. * dedup filter query definition Signed-off-by: Sam Lock <sam@swlock.co.uk> * moved regexp cache to util pkg. made scope match use regexp also Signed-off-by: Sam Lock <sam@swlock.co.uk> * made attribute naming more consistent Signed-off-by: Sam Lock <sam@swlock.co.uk> * use client specific model for filter policies options Signed-off-by: Sam Lock <sam@swlock.co.uk> * make generate Signed-off-by: Sam Lock <sam@swlock.co.uk> * add post-filtering for non-regexp compatible DBs. Plus some test refactoring Signed-off-by: Sam Lock <sam@swlock.co.uk> * minor log wording changes Signed-off-by: Sam Lock <sam@swlock.co.uk> * make generate and make lint Signed-off-by: Sam Lock <sam@swlock.co.uk> * merged FilterPolicies into preexisting ListPolicies RPC Signed-off-by: Sam Lock <sam@swlock.co.uk> * docs update Signed-off-by: Sam Lock <sam@swlock.co.uk> * minor touches Signed-off-by: Sam Lock <sam@swlock.co.uk> * use case insensitive regexp Signed-off-by: Sam Lock <sam@swlock.co.uk> * Apply suggestions from code review Co-authored-by: Charith Ellawala <charithe@users.noreply.github.com> Signed-off-by: Sam Lock <sam@swlock.co.uk> * use RWLock in regex cache Signed-off-by: Sam Lock <sam@swlock.co.uk> * version now supports regex too Signed-off-by: Sam Lock <sam@swlock.co.uk> * fixes for review comments Signed-off-by: Sam Lock <sam@swlock.co.uk> --------- Signed-off-by: Sam Lock <sam@swlock.co.uk> Co-authored-by: Charith Ellawala <charithe@users.noreply.github.com>
cerbos · Jun 15, 2023 · c2fcf27 · c2fcf27
1 parent fb98b74
commit c2fcf27
Show file tree

Hide file tree

Showing 35 changed files with 791 additions and 148 deletions.
diff --git a/api/genpb/cerbos/request/v1/hashpb_helpers.pb.go b/api/genpb/cerbos/request/v1/hashpb_helpers.pb.go
diff --git a/api/genpb/cerbos/request/v1/request.pb.go b/api/genpb/cerbos/request/v1/request.pb.go
diff --git a/api/genpb/cerbos/request/v1/request.pb.validate.go b/api/genpb/cerbos/request/v1/request.pb.validate.go
diff --git a/api/genpb/cerbos/request/v1/request_vtproto.pb.go b/api/genpb/cerbos/request/v1/request_vtproto.pb.go
diff --git a/api/public/cerbos/request/v1/request.proto b/api/public/cerbos/request/v1/request.proto
@@ -521,6 +521,18 @@ message ListPoliciesRequest {
     (google.api.field_behavior) = OPTIONAL,
     (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {description: "Include disabled policies"}
   ];
+  string name_regexp = 2 [
+    (google.api.field_behavior) = OPTIONAL,
+    (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {description: "Filter policies by name with regexp"}
+  ];
+  string scope_regexp = 3 [
+    (google.api.field_behavior) = OPTIONAL,
+    (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {description: "Filter policies by scope with regexp"}
+  ];
+  string version_regexp = 4 [
+    (google.api.field_behavior) = OPTIONAL,
+    (grpc.gateway.protoc_gen_openapiv2.options.openapiv2_field) = {description: "Filter policies by version with regexp"}
+  ];
 }
 
 message GetPolicyRequest {

diff --git a/client/admin.go b/client/admin.go
@@ -177,9 +177,7 @@ func (c *GrpcAdminClient) auditLogs(ctx context.Context, opts AuditLogOptions) (
 }
 
 func (c *GrpcAdminClient) ListPolicies(ctx context.Context, opts ...ListPoliciesOption) ([]string, error) {
-	req := &requestv1.ListPoliciesRequest{
-		IncludeDisabled: false,
-	}
+	req := &requestv1.ListPoliciesRequest{}
 	for _, opt := range opts {
 		opt(req)
 	}

diff --git a/client/admin_test.go b/client/admin_test.go
@@ -16,10 +16,12 @@ import (
 	"google.golang.org/grpc"
 
 	auditv1 "github.com/cerbos/cerbos/api/genpb/cerbos/audit/v1"
+	requestv1 "github.com/cerbos/cerbos/api/genpb/cerbos/request/v1"
 	responsev1 "github.com/cerbos/cerbos/api/genpb/cerbos/response/v1"
 	svcv1 "github.com/cerbos/cerbos/api/genpb/cerbos/svc/v1"
 	"github.com/cerbos/cerbos/client/testutil"
 	"github.com/cerbos/cerbos/internal/namer"
+	"github.com/cerbos/cerbos/internal/storage"
 	"github.com/cerbos/cerbos/internal/test"
 )
 
@@ -86,7 +88,9 @@ func TestAuditLogs(t *testing.T) {
 	})
 }
 
-func TestListPolicies(t *testing.T) {
+func setUpAdminClientAndPolicySet(t *testing.T) (AdminClient, *PolicySet) {
+	t.Helper()
+
 	const (
 		adminUsername = "cerbos"
 		adminPassword = "cerbosAdmin"
@@ -117,6 +121,12 @@ func TestListPolicies(t *testing.T) {
 	require.NoError(t, err)
 	require.NoError(t, ac.AddOrUpdatePolicy(context.Background(), ps))
 
+	return ac, ps
+}
+
+func TestListPolicies(t *testing.T) {
+	ac, ps := setUpAdminClientAndPolicySet(t)
+
 	t.Run("should get the list of policies", func(t *testing.T) {
 		have, err := ac.ListPolicies(context.Background(), WithIncludeDisabled())
 		require.NoError(t, err)
@@ -144,3 +154,56 @@ func TestListPolicies(t *testing.T) {
 		}
 	})
 }
+
+func TestFilterPolicies(t *testing.T) {
+	ac, ps := setUpAdminClientAndPolicySet(t)
+
+	testFilter := func(filterOpts ...ListPoliciesOption) {
+		t.Helper()
+
+		filterOpts = append(filterOpts, WithIncludeDisabled())
+
+		have, err := ac.ListPolicies(context.Background(), filterOpts...)
+		require.NoError(t, err)
+		require.NotEmpty(t, have)
+
+		// Bit of gymnastics to convert the client friendly filterOpts to backend-recognised params
+		r := &requestv1.ListPoliciesRequest{}
+		for _, opt := range filterOpts {
+			opt(r)
+		}
+		params := storage.ListPolicyIDsParams{
+			IncludeDisabled: r.IncludeDisabled,
+			NameRegexp:      r.NameRegexp,
+			ScopeRegexp:     r.ScopeRegexp,
+			VersionRegexp:   r.VersionRegexp,
+		}
+
+		policyList := test.FilterPolicies(t, ps.GetPolicies(), params)
+		want := make([]string, len(policyList))
+		for i, p := range policyList {
+			want[i] = namer.PolicyKey(p)
+		}
+		require.ElementsMatch(t, want, have)
+	}
+
+	t.Run("should get the list of filtered policies by name", func(t *testing.T) {
+		testFilter(WithNameRegexp(".*request$"))
+	})
+
+	t.Run("should get the list of filtered policies by scope", func(t *testing.T) {
+		testFilter(WithScopeRegexp("acme"))
+	})
+
+	t.Run("should get the list of filtered policies by version", func(t *testing.T) {
+		testFilter(WithVersionRegexp("20210210"))
+	})
+
+	t.Run("should get the list of filtered policies by all", func(t *testing.T) {
+		testFilter(
+			WithNameRegexp(".*(leave|equipment)_[rw]equest$"),
+			WithScopeRegexp("^acme"),
+			WithVersionRegexp("default$"),
+		)
+	})
+}
diff --git a/client/model.go b/client/model.go
@@ -1259,3 +1259,21 @@ func WithIncludeDisabled() ListPoliciesOption {
 		request.IncludeDisabled = true
 	}
 }
+
+func WithNameRegexp(re string) ListPoliciesOption {
+	return func(request *requestv1.ListPoliciesRequest) {
+		request.NameRegexp = re
+	}
+}
+
+func WithScopeRegexp(re string) ListPoliciesOption {
+	return func(request *requestv1.ListPoliciesRequest) {
+		request.ScopeRegexp = re
+	}
+}
+
+func WithVersionRegexp(v string) ListPoliciesOption {
+	return func(request *requestv1.ListPoliciesRequest) {
+		request.VersionRegexp = v
+	}
+}
diff --git a/docs/modules/api/pages/admin_api.adoc b/docs/modules/api/pages/admin_api.adoc
@@ -131,10 +131,12 @@ GET /admin/policies
 
 NOTE: This endpoint is still under development and should be considered unstable.
 
-Issue a GET request to the endpoint to list the policies available in the store.
+Issue a GET request to the endpoint to list the policies available in the store. If the policy store supports filtering, you can optionally pass filter parameters to reduce the result set.
 
 Use `includeDisabled=true` query parameter in order to include disabled policies in the response.
 
+Use `nameRegexp`, `scopeRegexp` and `versionRegexp` to filter using the policy name, scope or version with case insensitive regular expressions.
+
 [source,shell]
 ----
 curl -k -u cerbos:cerbosAdmin \
@@ -144,7 +146,7 @@ curl -k -u cerbos:cerbosAdmin \
 [source,shell]
 ----
 curl -k -u cerbos:cerbosAdmin \
-    'https://localhost:3592/admin/policies?pretty&includeDisabled=true'
+    'https://localhost:3592/admin/policies?pretty&includeDisabled=true&nameRegexp=%5Efoo&scopeRegexp=bar%24&versionRegexp=default'
 ----
 
 === Get Policies

diff --git a/internal/compile/manager_test.go b/internal/compile/manager_test.go
@@ -288,7 +288,7 @@ func (ms *MockStore) Delete(ctx context.Context, ids ...namer.ModuleID) error {
 	return args.Error(0)
 }
 
-func (ms *MockStore) ListPolicyIDs(ctx context.Context, _ bool) ([]string, error) {
+func (ms *MockStore) ListPolicyIDs(ctx context.Context, _ storage.ListPolicyIDsParams) ([]string, error) {
 	args := ms.MethodCalled("ListPolicyIDs", ctx)
 	if res := args.Get(0); res == nil {
 		return nil, args.Error(0)

diff --git a/internal/storage/blob/store.go b/internal/storage/blob/store.go
@@ -314,7 +314,7 @@ func (s *Store) GetDependents(_ context.Context, ids ...namer.ModuleID) (map[nam
 	return s.idx.GetDependents(ids...)
 }
 
-func (s *Store) ListPolicyIDs(ctx context.Context, _ bool) ([]string, error) {
+func (s *Store) ListPolicyIDs(ctx context.Context, _ storage.ListPolicyIDsParams) ([]string, error) {
 	return s.idx.ListPolicyIDs(ctx)
 }
 

diff --git a/internal/storage/bundle/bundle.go b/internal/storage/bundle/bundle.go
@@ -15,6 +15,7 @@ import (
 
 	runtimev1 "github.com/cerbos/cerbos/api/genpb/cerbos/runtime/v1"
 	"github.com/cerbos/cerbos/internal/namer"
+	"github.com/cerbos/cerbos/internal/storage"
 	"github.com/cerbos/cloud-api/credentials"
 	bundlev1 "github.com/cerbos/cloud-api/genpb/cerbos/cloud/bundle/v1"
 	"github.com/spf13/afero"
@@ -201,7 +202,7 @@ func (b *Bundle) GetPolicySet(_ context.Context, id namer.ModuleID) (*runtimev1.
 	return rps, nil
 }
 
-func (b *Bundle) ListPolicyIDs(_ context.Context, _ bool) ([]string, error) {
+func (b *Bundle) ListPolicyIDs(_ context.Context, _ storage.ListPolicyIDsParams) ([]string, error) {
 	output := make([]string, len(b.manifest.PolicyIndex))
 
 	i := 0

diff --git a/internal/storage/bundle/local_source.go b/internal/storage/bundle/local_source.go
@@ -119,9 +119,9 @@ func (ls *LocalSource) Driver() string {
 	return DriverName
 }
 
-func (ls *LocalSource) ListPolicyIDs(ctx context.Context, includeDisabled bool) (ids []string, err error) {
+func (ls *LocalSource) ListPolicyIDs(ctx context.Context, params storage.ListPolicyIDsParams) (ids []string, err error) {
 	ls.mu.RLock()
-	ids, err = ls.bundle.ListPolicyIDs(ctx, includeDisabled)
+	ids, err = ls.bundle.ListPolicyIDs(ctx, params)
 	ls.mu.RUnlock()
 	return ids, err
 }