implement provider arg option caching

go.mod

@@ -149,7 +149,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/lambda v1.23.4
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.27.0
 	github.com/briandowns/spinner v1.18.1
-	github.com/common-fate/ddb v0.12.0
+	github.com/common-fate/ddb v0.13.0
 	github.com/common-fate/testvault v0.1.0
 	github.com/fatih/color v1.13.0
 	github.com/go-chi/cors v1.2.1

go.sum

@@ -265,6 +265,8 @@ github.com/common-fate/ddb v0.11.0 h1:huNJpr/fk2DuAfWEjlA3jyDZOSxYkr+t6yYW73FyiB
 github.com/common-fate/ddb v0.11.0/go.mod h1:Z4JJVRI62wOGCef0D+ZA6W21GfdQvchw9X/Olf0uXtg=
 github.com/common-fate/ddb v0.12.0 h1:YjjkLconEO0q2sVn901eHrkiKp6iW/YkFoljN5RmgTE=
 github.com/common-fate/ddb v0.12.0/go.mod h1:Z4JJVRI62wOGCef0D+ZA6W21GfdQvchw9X/Olf0uXtg=
+github.com/common-fate/ddb v0.13.0 h1:0WkelmqTwtJSoXCgQB2GAJXxr+qM8eqhblHmgY6mqvg=
+github.com/common-fate/ddb v0.13.0/go.mod h1:Z4JJVRI62wOGCef0D+ZA6W21GfdQvchw9X/Olf0uXtg=
 github.com/common-fate/iso8601 v1.0.2 h1:gl6iNbE8TnJzg+lYJxPNYOkF+oHYde2K2s2i+5o4yHE=
 github.com/common-fate/iso8601 v1.0.2/go.mod h1:DU4mvUEkkWZUUSJq2aCuNqM1luSb0Pwyb2dLzXS+img=
 github.com/common-fate/testvault v0.1.0 h1:XVhbmcNySGIA203FywYW7wL44Mlcg23UUek3bdg5tzQ=

openapi.yml

@@ -686,7 +686,13 @@ paths:
         "500":
           $ref: "#/components/responses/ErrorResponse"
       operationId: list-provider-arg-options
-      description: ""
+      description: "Returns the options for a particular Access Provider argument. The options may be cached. To refresh the cache, pass the `refresh` query parameter."
+      parameters:
+        - schema:
+            type: boolean
+          in: query
+          name: refresh
+          description: invalidate the cache and refresh the provider's options.
 components:
   schemas:
     User:

pkg/api/provider.go

@@ -1,12 +1,18 @@
 package api
 
 import (
+	"context"
 	"errors"
+	"fmt"
 	"net/http"
 
 	"github.com/common-fate/apikit/apio"
 	"github.com/common-fate/apikit/logger"
+	"github.com/common-fate/ddb"
 	ahTypes "github.com/common-fate/granted-approvals/accesshandler/pkg/types"
+	"github.com/common-fate/granted-approvals/pkg/cache"
+	"github.com/common-fate/granted-approvals/pkg/storage"
+	"github.com/common-fate/granted-approvals/pkg/types"
 )
 
 func (a *API) ListProviders(w http.ResponseWriter, r *http.Request) {
@@ -91,30 +97,147 @@ func (a *API) GetProviderArgs(w http.ResponseWriter, r *http.Request, providerId
 	}
 }
 
-func (a *API) ListProviderArgOptions(w http.ResponseWriter, r *http.Request, providerId string, argId string) {
+// List provider arg options
+// (GET /api/v1/admin/providers/{providerId}/args/{argId}/options)
+func (a *API) ListProviderArgOptions(w http.ResponseWriter, r *http.Request, providerId string, argId string, params types.ListProviderArgOptionsParams) {
+	if params.Refresh != nil && *params.Refresh {
+		a.refreshProviderArgOptions(w, r, providerId, argId)
+	} else {
+		a.getCachedProviderArgOptions(w, r, providerId, argId)
+	}
+}
+
+// getCachedProviderArgOptions handles the case where we fetch arg options from the DynamoDB cache.
+// If cached options aren't present it falls back to refetching options from the Access Handler.
+// If options are refetched, the cache is updated.
+func (a *API) getCachedProviderArgOptions(w http.ResponseWriter, r *http.Request, providerId string, argId string) {
+	ctx := r.Context()
+
+	q := storage.GetProviderOptions{
+		ProviderID: providerId,
+		ArgID:      argId,
+	}
+
+	_, err := a.DB.Query(ctx, &q)
+	if err != nil && err != ddb.ErrNoItems {
+		apio.Error(ctx, w, err)
+		return
+	}
+
+	var res types.ArgOptionsResponse
+
+	if err == ddb.ErrNoItems {
+		// we don't have any cached, so try and refetch them.
+		res, err = a.fetchProviderOptions(ctx, providerId, argId)
+		if err != nil {
+			apio.Error(ctx, w, err)
+			return
+		}
+		var cachedOpts []ddb.Keyer
+		for _, o := range res.Options {
+			cachedOpts = append(cachedOpts, &cache.ProviderOption{
+				Provider: providerId,
+				Arg:      argId,
+				Label:    o.Label,
+				Value:    o.Value,
+			})
+		}
+		err = a.DB.PutBatch(ctx, cachedOpts...)
+		if err != nil {
+			apio.Error(ctx, w, err)
+			return
+		}
+	} else {
+		// we have cached options
+		res = types.ArgOptionsResponse{
+			HasOptions: true,
+		}
+		for _, o := range q.Result {
+			res.Options = append(res.Options, ahTypes.Option{
+				Label: o.Label,
+				Value: o.Value,
+			})
+		}
+	}
+
+	// return the argument options back to the client
+	apio.JSON(ctx, w, res, http.StatusOK)
+}
+
+// refreshProviderArgOptions deletes any cached options and then refetches them from the Access Handler.
+// It updates the cached options.
+func (a *API) refreshProviderArgOptions(w http.ResponseWriter, r *http.Request, providerId string, argId string) {
 	ctx := r.Context()
 
-	res, err := a.AccessHandlerClient.ListProviderArgOptionsWithResponse(ctx, providerId, argId)
+	// delete any existing options
+	q := storage.GetProviderOptions{
+		ProviderID: providerId,
+		ArgID:      argId,
+	}
+
+	_, err := a.DB.Query(ctx, &q)
+	if err != nil && err != ddb.ErrNoItems {
+		apio.Error(ctx, w, err)
+		return
+	}
+	var items []ddb.Keyer
+	for _, row := range q.Result {
+		po := row
+		items = append(items, &po)
+	}
+	err = a.DB.DeleteBatch(ctx, items...)
+	if err != nil {
+		apio.Error(ctx, w, err)
+		return
+	}
+
+	// fetch new options
+	res, err := a.fetchProviderOptions(ctx, providerId, argId)
 	if err != nil {
 		apio.Error(ctx, w, err)
 		return
 	}
+
+	// update the cache
+	var cachedOpts []ddb.Keyer
+	for _, o := range res.Options {
+		cachedOpts = append(cachedOpts, &cache.ProviderOption{
+			Provider: providerId,
+			Arg:      argId,
+			Label:    o.Label,
+			Value:    o.Value,
+		})
+	}
+	err = a.DB.PutBatch(ctx, cachedOpts...)
+	if err != nil {
+		apio.Error(ctx, w, err)
+		return
+	}
+
+	// return the argument options back to the client
+	apio.JSON(ctx, w, res, http.StatusOK)
+}
+
+func (a *API) fetchProviderOptions(ctx context.Context, providerID, argID string) (types.ArgOptionsResponse, error) {
+	res, err := a.AccessHandlerClient.ListProviderArgOptionsWithResponse(ctx, providerID, argID)
+	if err != nil {
+		return types.ArgOptionsResponse{}, err
+	}
 	code := res.StatusCode()
 	switch code {
 	case 200:
-		apio.JSON(ctx, w, res.JSON200, code)
-		return
+		opts := types.ArgOptionsResponse{
+			HasOptions: res.JSON200.HasOptions,
+			Options:    res.JSON200.Options,
+		}
+		return opts, nil
 	case 404:
-		apio.JSON(ctx, w, res.JSON404, code)
-		return
+		err := errors.New("provider not found")
+		return types.ArgOptionsResponse{}, apio.NewRequestError(err, http.StatusNotFound)
 	case 500:
-		apio.JSON(ctx, w, res.JSON500, code)
-		return
+		return types.ArgOptionsResponse{}, errors.New(*res.JSON500.Error)
 	default:
-		if err != nil {
-			logger.Get(ctx).Errorw("unhandled access handler response", "response", string(res.Body))
-			apio.Error(ctx, w, errors.New("unhandled response code"))
-			return
-		}
+		logger.Get(ctx).Errorw("unhandled access handler response", "response", string(res.Body))
+		return types.ArgOptionsResponse{}, fmt.Errorf("unhandled response code: %d", code)
 	}
 }

pkg/cache/provider_option.go

@@ -0,0 +1,28 @@
+// Package cache stores provider information in DynamoDB so we
+// don't need to call slow external APIs, like AWS SSO,
+// every time a user is setting up an Access Rule or making an
+// Access Request.
+package cache
+
+import (
+	"github.com/common-fate/ddb"
+	"github.com/common-fate/granted-approvals/pkg/storage/keys"
+)
+
+// ProviderOption is an argument option that we've cached
+// from an Access Provider in DynamoDB.
+type ProviderOption struct {
+	Provider string `json:"provider" dynamodbav:"provider"`
+	Arg      string `json:"arg" dynamodbav:"arg"`
+	Label    string `json:"label" dynamodbav:"label"`
+	Value    string `json:"value" dynamodbav:"value"`
+}
+
+func (r *ProviderOption) DDBKeys() (ddb.Keys, error) {
+	keys := ddb.Keys{
+		PK: keys.ProviderOption.PK1,
+		SK: keys.ProviderOption.SK1(r.Provider, r.Arg, r.Value),
+	}
+
+	return keys, nil
+}

pkg/storage/get_provider_options.go

@@ -0,0 +1,37 @@
+package storage
+
+import (
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb"
+	"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
+	"github.com/common-fate/ddb"
+	"github.com/common-fate/granted-approvals/pkg/cache"
+	"github.com/common-fate/granted-approvals/pkg/storage/keys"
+)
+
+type GetProviderOptions struct {
+	ProviderID string
+	ArgID      string
+	Result     []cache.ProviderOption
+}
+
+func (q *GetProviderOptions) BuildQuery() (*dynamodb.QueryInput, error) {
+	qi := dynamodb.QueryInput{
+		KeyConditionExpression: aws.String("PK = :pk1 and begins_with(SK, :sk1)"),
+		ExpressionAttributeValues: map[string]types.AttributeValue{
+			":pk1": &types.AttributeValueMemberS{Value: keys.ProviderOption.PK1},
+			":sk1": &types.AttributeValueMemberS{Value: q.ProviderID + "#" + q.ArgID},
+		},
+	}
+	return &qi, nil
+}
+
+func (q *GetProviderOptions) UnmarshalQueryOutput(out *dynamodb.QueryOutput) error {
+	if len(out.Items) == 0 {
+		return ddb.ErrNoItems
+	}
+
+	return attributevalue.UnmarshalListOfMaps(out.Items, &q.Result)
+
+}

pkg/storage/get_provider_options_test.go

@@ -0,0 +1,36 @@
+package storage
+
+import (
+	"testing"
+
+	"github.com/common-fate/ddb"
+	"github.com/common-fate/ddb/ddbtest"
+	"github.com/common-fate/granted-approvals/pkg/cache"
+)
+
+func TestGetProviderOptions(t *testing.T) {
+	db := newTestingStorage(t)
+
+	po := cache.ProviderOption{
+		Provider: "test",
+		Arg:      "test",
+		Label:    "test",
+		Value:    "test",
+	}
+	ddbtest.PutFixtures(t, db, &po)
+
+	tc := []ddbtest.QueryTestCase{
+		{
+			Name:  "ok",
+			Query: &GetProviderOptions{ProviderID: "test", ArgID: "test"},
+			Want:  &GetProviderOptions{ProviderID: "test", ArgID: "test", Result: []cache.ProviderOption{po}},
+		},
+		{
+			Name:    "not found",
+			Query:   &GetProviderOptions{ProviderID: "somethingelse", ArgID: "test"},
+			WantErr: ddb.ErrNoItems,
+		},
+	}
+
+	ddbtest.RunQueryTests(t, db, tc)
+}

pkg/storage/keys/provideroption.go

@@ -0,0 +1,13 @@
+package keys
+
+const ProviderOptionKey = "PROVIDER_OPTION#"
+
+type providerOptionKeys struct {
+	PK1 string
+	SK1 func(providerID, argID, value string) string
+}
+
+var ProviderOption = providerOptionKeys{
+	PK1: ProviderOptionKey,
+	SK1: func(providerID, argID, value string) string { return providerID + "#" + argID + "#" + value },
+}