From 2bd37a86139ef2ca13fbc1dad0184ac646101d2b Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Mon, 7 Aug 2023 14:49:07 +0200 Subject: [PATCH 1/4] Also log status when getting a change --- cmd/getchange.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/getchange.go b/cmd/getchange.go index 3d43065a..98d5c887 100644 --- a/cmd/getchange.go +++ b/cmd/getchange.go @@ -90,6 +90,7 @@ func GetChange(signals chan os.Signal, ready chan bool) int { log.WithContext(ctx).WithFields(log.Fields{ "change-uuid": uuid.UUID(response.Msg.Change.Metadata.UUID), "change-created": response.Msg.Change.Metadata.CreatedAt.AsTime(), + "change-status": response.Msg.Change.Metadata.Status.String(), "change-name": response.Msg.Change.Properties.Title, "change-description": response.Msg.Change.Properties.Description, }).Info("found change") From 210c1bf9b5caaed8b7910996a77475bbaaf877c1 Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Mon, 7 Aug 2023 14:50:55 +0200 Subject: [PATCH 2/4] Fix error handling when streaming UpdateChangingItems responses --- cmd/submitplan.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/submitplan.go b/cmd/submitplan.go index 3dbe4d9c..5588bdb3 100644 --- a/cmd/submitplan.go +++ b/cmd/submitplan.go @@ -459,11 +459,6 @@ func SubmitPlan(signals chan os.Signal, ready chan bool) int { last_log := time.Now() first_log := true for resultStream.Receive() { - if resultStream.Err() != nil { - log.WithContext(ctx).WithFields(lf).WithError(err).Error("error streaming results") - return 1 - } - msg := resultStream.Msg() // log the first message and at most every 250ms during discovery @@ -475,6 +470,10 @@ func SubmitPlan(signals chan os.Signal, ready chan bool) int { first_log = false } } + if resultStream.Err() != nil { + log.WithContext(ctx).WithFields(lf).WithError(err).Error("error streaming results") + return 1 + } changeUrl := fmt.Sprintf("%v/changes/%v", viper.GetString("frontend"), changeUuid) log.WithContext(ctx).WithFields(lf).WithField("change-url", changeUrl).Info("change ready") From 18f41919b2eee41333c97004f3f90924b93dde8c Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Wed, 9 Aug 2023 15:07:46 +0200 Subject: [PATCH 3/4] Only request minimal scopes for each operation This requires https://github.com/overmindtech/api-server/pull/337 to be deployed and in use by customers to avoid running into the 'no user in database' case. --- cmd/createbookmark.go | 2 +- cmd/endchange.go | 2 +- cmd/getaffectedbookmarks.go | 2 +- cmd/getbookmark.go | 2 +- cmd/getchange.go | 2 +- cmd/getsnapshot.go | 2 +- cmd/request.go | 2 +- cmd/root.go | 4 ++-- cmd/startchange.go | 2 +- cmd/submitplan.go | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cmd/createbookmark.go b/cmd/createbookmark.go index a685c53e..9d7550f5 100644 --- a/cmd/createbookmark.go +++ b/cmd/createbookmark.go @@ -66,7 +66,7 @@ func CreateBookmark(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:write"}, signals) if err != nil { log.WithContext(ctx).WithError(err).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/endchange.go b/cmd/endchange.go index 40592dbc..b958a796 100644 --- a/cmd/endchange.go +++ b/cmd/endchange.go @@ -52,7 +52,7 @@ func EndChange(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:write"}, signals) if err != nil { log.WithContext(ctx).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/getaffectedbookmarks.go b/cmd/getaffectedbookmarks.go index c36e6f01..cb8cb02e 100644 --- a/cmd/getaffectedbookmarks.go +++ b/cmd/getaffectedbookmarks.go @@ -70,7 +70,7 @@ func GetAffectedBookmarks(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:read"}, signals) if err != nil { log.WithContext(ctx).WithError(err).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/getbookmark.go b/cmd/getbookmark.go index 42c5d8b8..6fc64ed6 100644 --- a/cmd/getbookmark.go +++ b/cmd/getbookmark.go @@ -60,7 +60,7 @@ func GetBookmark(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:read"}, signals) if err != nil { log.WithContext(ctx).WithError(err).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/getchange.go b/cmd/getchange.go index 98d5c887..a2616539 100644 --- a/cmd/getchange.go +++ b/cmd/getchange.go @@ -54,7 +54,7 @@ func GetChange(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:read"}, signals) if err != nil { log.WithContext(ctx).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/getsnapshot.go b/cmd/getsnapshot.go index 7ddf3a9f..b48c395f 100644 --- a/cmd/getsnapshot.go +++ b/cmd/getsnapshot.go @@ -59,7 +59,7 @@ func GetSnapshot(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:read"}, signals) if err != nil { log.WithContext(ctx).WithError(err).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/request.go b/cmd/request.go index 03d4ceec..563f3f11 100644 --- a/cmd/request.go +++ b/cmd/request.go @@ -74,7 +74,7 @@ func Request(signals chan os.Signal, ready chan bool) int { lf := log.Fields{} - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"explore:read"}, signals) if err != nil { log.WithContext(ctx).WithFields(lf).WithField("api-key-url", viper.GetString("api-key-url")).WithError(err).Error("failed to authenticate") return 1 diff --git a/cmd/root.go b/cmd/root.go index 8f30eb98..81921bbf 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -57,7 +57,7 @@ func Execute() { } // ensureToken -func ensureToken(ctx context.Context, signals chan os.Signal) (context.Context, error) { +func ensureToken(ctx context.Context, requiredScopes []string, signals chan os.Signal) (context.Context, error) { // get a token from the api key if present if viper.GetString("api-key") != "" { log.WithContext(ctx).Debug("using provided token for authentication") @@ -97,7 +97,7 @@ func ensureToken(ctx context.Context, signals chan os.Signal) (context.Context, // Authenticate using the oauth resource owner password flow config := oauth2.Config{ ClientID: viper.GetString("auth0-client-id"), - Scopes: []string{"openid", "profile", "email", "gateway:stream", "request:send", "reverselink:request", "account:read", "source:read", "source:write", "api:read", "api:write", "gateway:objects"}, + Scopes: requiredScopes, Endpoint: oauth2.Endpoint{ AuthURL: fmt.Sprintf("https://%v/authorize", viper.GetString("auth0-domain")), TokenURL: fmt.Sprintf("https://%v/oauth/token", viper.GetString("auth0-domain")), diff --git a/cmd/startchange.go b/cmd/startchange.go index abe16bf4..d6589a45 100644 --- a/cmd/startchange.go +++ b/cmd/startchange.go @@ -52,7 +52,7 @@ func StartChange(signals chan os.Signal, ready chan bool) int { )) defer span.End() - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:write"},signals) if err != nil { log.WithContext(ctx).WithFields(log.Fields{ "url": viper.GetString("url"), diff --git a/cmd/submitplan.go b/cmd/submitplan.go index 5588bdb3..e2bda661 100644 --- a/cmd/submitplan.go +++ b/cmd/submitplan.go @@ -206,7 +206,7 @@ func SubmitPlan(signals chan os.Signal, ready chan bool) int { lf := log.Fields{} - ctx, err = ensureToken(ctx, signals) + ctx, err = ensureToken(ctx, []string{"changes:write"}, signals) if err != nil { log.WithContext(ctx).WithFields(lf).WithField("api-key-url", viper.GetString("api-key-url")).WithError(err).Error("failed to authenticate") return 1 From 8fc4741e52d748eb4bfa389de2b5c50604da8f2e Mon Sep 17 00:00:00 2001 From: David Schmitt Date: Wed, 9 Aug 2023 15:09:31 +0200 Subject: [PATCH 4/4] Add cloudfront datamaps --- cmd/datamaps/awssource.go | 72 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/cmd/datamaps/awssource.go b/cmd/datamaps/awssource.go index 14a02b7a..96982a84 100644 --- a/cmd/datamaps/awssource.go +++ b/cmd/datamaps/awssource.go @@ -45,6 +45,78 @@ var AwssourceData = map[string][]TfMapData{ Scope: "*", }, }, + "aws_cloudfront_Streamingdistribution": { + { + Type: "cloudfront-streaming-distribution", + Method: sdp.QueryMethod_SEARCH, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_cloudfront_cache_policy": { + { + Type: "cloudfront-cache-policy", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_cloudfront_distribution": { + { + Type: "cloudfront-distribution", + Method: sdp.QueryMethod_SEARCH, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_cloudfront_function": { + { + Type: "cloudfront-function", + Method: sdp.QueryMethod_GET, + QueryField: "name", + Scope: "*", + }, + }, + "aws_cloudfront_key_group": { + { + Type: "cloudfront-key-group", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_cloudfront_origin_access_control": { + { + Type: "cloudfront-origin-access-control", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_cloudfront_origin_request_policy": { + { + Type: "cloudfront-origin-request-policy", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, + "aws_cloudfront_realtime_log_config": { + { + Type: "cloudfront-realtime-log-config", + Method: sdp.QueryMethod_SEARCH, + QueryField: "arn", + Scope: "*", + }, + }, + "aws_cloudfront_response_headers_policy": { + { + Type: "cloudfront-response-headers-policy", + Method: sdp.QueryMethod_GET, + QueryField: "id", + Scope: "*", + }, + }, "aws_cloudwatch_metric_alarm": { { Type: "cloudwatch-alarm",