From b3a7eaa51f8d287fad0b6c3200dc9490ac9cc93d Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Mon, 22 Aug 2022 11:18:23 -0500 Subject: [PATCH 1/8] feat(storage): find google access id when using impersonated creds --- storage/bucket.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 87789f3bc29a..763fd53a56da 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "reflect" + "strings" "time" "cloud.google.com/go/compute/metadata" @@ -263,17 +264,27 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { if b.c.creds != nil && len(b.c.creds.JSON) > 0 { var sa struct { - ClientEmail string `json:"client_email"` + ClientEmail string `json:"client_email"` + SAImpersonationURL string `json:"service_account_impersonation_url"` + CredType string `json:"type"` } + err := json.Unmarshal(b.c.creds.JSON, &sa) - if err == nil && sa.ClientEmail != "" { - return sa.ClientEmail, nil - } else if err != nil { + if err != nil { returnErr = err + } else if sa.CredType == "impersonated_service_account" { + start, end := strings.LastIndex(sa.SAImpersonationURL, "/"), strings.LastIndex(sa.SAImpersonationURL, ":") + + if end <= start { + returnErr = errors.New("error parsing impersonated service account") + } else { + return sa.SAImpersonationURL[start+1 : end], nil + } + } else if sa.ClientEmail != "" { + return sa.ClientEmail, nil } else { - returnErr = errors.New("storage: empty client email in credentials") + returnErr = errors.New("no client email in credentials. Please set GoogleAccessID.") } - } // Don't error out if we can't unmarshal, fallback to GCE check. From 6bc33daaea0f5db8d4daec2d3796932eaeab6f7b Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Tue, 30 Aug 2022 15:08:32 -0500 Subject: [PATCH 2/8] feat(storage): auto-detect ID for impersonated creds for SignURL --- storage/bucket.go | 71 ++++++++++++++++------------------ storage/bucket_test.go | 88 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+), 37 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index f79e041a03ad..9c56293ff150 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -276,14 +276,14 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { start, end := strings.LastIndex(sa.SAImpersonationURL, "/"), strings.LastIndex(sa.SAImpersonationURL, ":") if end <= start { - returnErr = errors.New("error parsing impersonated service account") + returnErr = errors.New("error parsing impersonated service account credentials") } else { return sa.SAImpersonationURL[start+1 : end], nil } - } else if sa.ClientEmail != "" { + } else if sa.CredType == "service_account" && sa.ClientEmail != "" { return sa.ClientEmail, nil } else { - returnErr = errors.New("no client email in credentials. Please set GoogleAccessID.") + returnErr = errors.New("unable to parse credentials") } } @@ -295,7 +295,7 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { } else if err != nil { returnErr = err } else { - returnErr = errors.New("got empty email from GCE metadata service") + returnErr = errors.New("empty email from GCE metadata service") } } @@ -621,12 +621,7 @@ const ( // // All configured conditions must be met for the associated action to be taken. type LifecycleCondition struct { - // AllObjects is used to select all objects in a bucket by - // setting AgeInDays to 0. - AllObjects bool - // AgeInDays is the age of the object in days. - // If you want to set AgeInDays to `0` use AllObjects set to `true`. AgeInDays int64 // CreatedBefore is the time the object was created. @@ -644,12 +639,10 @@ type LifecycleCondition struct { // DaysSinceCustomTime is the days elapsed since the CustomTime date of the // object. This condition can only be satisfied if CustomTime has been set. - // Note: Using `0` as the value will be ignored by the library and not sent to the API. DaysSinceCustomTime int64 // DaysSinceNoncurrentTime is the days elapsed since the noncurrent timestamp // of the object. This condition is relevant only for versioned objects. - // Note: Using `0` as the value will be ignored by the library and not sent to the API. DaysSinceNoncurrentTime int64 // Liveness specifies the object's liveness. Relevant only for versioned objects @@ -681,7 +674,6 @@ type LifecycleCondition struct { // If the value is N, this condition is satisfied when there are at least N // versions (including the live version) newer than this version of the // object. - // Note: Using `0` as the value will be ignored by the library and not sent to the API. NumNewerVersions int64 } @@ -1440,6 +1432,19 @@ func toCORSFromProto(rc []*storagepb.Bucket_Cors) []CORS { return out } +// Used to handle breaking change in Autogen Storage client OLM Age field +// from int64 to *int64 gracefully in the manual client +// TODO(#6240): Method should be removed once breaking change is made and introduced to this client +func setAgeCondition(age int64, ageField interface{}) { + c := reflect.ValueOf(ageField).Elem() + switch c.Kind() { + case reflect.Int64: + c.SetInt(age) + case reflect.Ptr: + c.Set(reflect.ValueOf(&age)) + } +} + func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { var rl raw.BucketLifecycle if len(l.Rules) == 0 { @@ -1461,15 +1466,7 @@ func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { }, } - // AllObjects takes precedent when both AllObjects and AgeInDays are set - // Rationale: If you've opted into using AllObjects, it makes sense that you - // understand the implications of how this option works with AgeInDays. - if r.Condition.AllObjects { - rr.Condition.Age = googleapi.Int64(0) - rr.Condition.ForceSendFields = []string{"Age"} - } else if r.Condition.AgeInDays > 0 { - rr.Condition.Age = googleapi.Int64(r.Condition.AgeInDays) - } + setAgeCondition(r.Condition.AgeInDays, &rr.Condition.Age) switch r.Condition.Liveness { case LiveAndArchived: @@ -1518,11 +1515,6 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle { }, } - // TODO(#6205): This may not be needed for gRPC - if r.Condition.AllObjects { - rr.Condition.AgeDays = proto.Int32(0) - } - switch r.Condition.Liveness { case LiveAndArchived: rr.Condition.IsLive = nil @@ -1546,6 +1538,21 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle { return &rl } +// Used to handle breaking change in Autogen Storage client OLM Age field +// from int64 to *int64 gracefully in the manual client +// TODO(#6240): Method should be removed once breaking change is made and introduced to this client +func getAgeCondition(ageField interface{}) int64 { + v := reflect.ValueOf(ageField) + if v.Kind() == reflect.Int64 { + return v.Interface().(int64) + } else if v.Kind() == reflect.Ptr { + if val, ok := v.Interface().(*int64); ok { + return *val + } + } + return 0 +} + func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { var l Lifecycle if rl == nil { @@ -1566,12 +1573,7 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { NumNewerVersions: rr.Condition.NumNewerVersions, }, } - if rr.Condition.Age != nil { - r.Condition.AgeInDays = *rr.Condition.Age - if *rr.Condition.Age == 0 { - r.Condition.AllObjects = true - } - } + r.Condition.AgeInDays = getAgeCondition(rr.Condition.Age) if rr.Condition.IsLive == nil { r.Condition.Liveness = LiveAndArchived @@ -1617,11 +1619,6 @@ func toLifecycleFromProto(rl *storagepb.Bucket_Lifecycle) Lifecycle { }, } - // TODO(#6205): This may not be needed for gRPC - if rr.GetCondition().GetAgeDays() == 0 { - r.Condition.AllObjects = true - } - if rr.GetCondition().IsLive == nil { r.Condition.Liveness = LiveAndArchived } else if rr.GetCondition().GetIsLive() { diff --git a/storage/bucket_test.go b/storage/bucket_test.go index 1a343d5c97b9..b8cc20c96a59 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -15,12 +15,14 @@ package storage import ( + "fmt" "testing" "time" "cloud.google.com/go/internal/testutil" "github.com/google/go-cmp/cmp" gax "github.com/googleapis/gax-go/v2" + "golang.org/x/oauth2/google" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" ) @@ -782,3 +784,89 @@ func TestBucketRetryer(t *testing.T) { }) } } + +func TestDetectDefaultGoogleAccessID(t *testing.T) { + testCases := []struct { + name string + serviceAccount string + creds func(string) string + expectSuccess bool + }{ + { + name: "impersonated creds", + serviceAccount: "default@my-project.iam.gserviceaccount.com", + creds: func(sa string) string { + return fmt.Sprintf(`{ + "delegates": [], + "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/%s:generateAccessToken", + "source_credentials": { + "client_id": "id", + "client_secret": "secret", + "refresh_token": "token", + "type": "authorized_user" + }, + "type": "impersonated_service_account" + }`, sa) + }, + expectSuccess: true, + }, + { + name: "gcloud ADC creds", + serviceAccount: "default@my-project.iam.gserviceaccount.com", + creds: func(sa string) string { + return fmt.Sprint(`{ + "client_id": "my-id.apps.googleusercontent.com", + "client_secret": "secret", + "quota_project_id": "", + "refresh_token": "token", + "type": "authorized_user" + }`) + }, + expectSuccess: false, + }, + { + name: "ADC private key", + serviceAccount: "default@my-project.iam.gserviceaccount.com", + creds: func(sa string) string { + return fmt.Sprintf(`{ + "type": "service_account", + "project_id": "my-project", + "private_key_id": "my1", + "private_key": "-----BEGIN PRIVATE KEY-----\nkey\n-----END PRIVATE KEY-----\n", + "client_email": "%s", + "client_id": "01", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "cert" + }`, sa) + }, + expectSuccess: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + bucket := BucketHandle{ + c: &Client{ + creds: &google.Credentials{ + JSON: []byte(tc.creds(tc.serviceAccount)), + }, + }, + name: "my-bucket", + } + + id, err := bucket.detectDefaultGoogleAccessID() + if tc.expectSuccess { + if err != nil { + t.Fatal(err) + } + if id != tc.serviceAccount { + t.Errorf("service account not found correctly; got: %s, want: %s", id, tc.serviceAccount) + } + } else if err == nil { + t.Error("expected error but detectDefaultGoogleAccessID did not return one") + } + }) + } +} From c0f489646c60cf2454f30ae5bb8a63eefe7ac421 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Tue, 30 Aug 2022 15:23:18 -0500 Subject: [PATCH 3/8] update singurl/postpolicy docs --- storage/bucket.go | 82 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 25 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 9c56293ff150..72dc20ee9719 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -160,22 +160,38 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) ( } // SignedURL returns a URL for the specified object. Signed URLs allow anyone -// access to a restricted resource for a limited time without needing a -// Google account or signing in. For more information about signed URLs, see -// https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication +// access to a restricted resource for a limited time without needing a Google +// account or signing in. +// For more information about signed URLs, see "[Overview of access control]." // -// This method only requires the Method and Expires fields in the specified -// SignedURLOptions opts to be non-nil. If not provided, it attempts to fill the -// GoogleAccessID and PrivateKey from the GOOGLE_APPLICATION_CREDENTIALS environment variable. -// If you are authenticating with a custom HTTP client, Service Account based -// auto-detection will be hindered. +// This method requires the Method and Expires fields in the specified +// SignedURLOptions to be non-nil. // -// If no private key is found, it attempts to use the GoogleAccessID to sign the URL. -// This requires the IAM Service Account Credentials API to be enabled -// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview) -// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. -// If you do not want these fields set for you, you may pass them in through opts or use -// SignedURL(bucket, name string, opts *SignedURLOptions) instead. +// If the GoogleAccessID and PrivateKey fields are not provided, they will be +// automatically detected when: +// - you are authenticated to the Storage Client with a service account's +// downloaded private key, either directly in code or by setting the +// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), +// - your application is running on Google Compute Engine (GCE), or +// - you are logged into [gcloud using application default credentials] +// with [impersonation enabled]. +// In some cases, you may not need to set PrivateKey but must set GoogleAccessID. +// GoogleAccessID should be set to a service account that will be used to attempt +// to sign the URL. This is true of cases where credentials are provided but not +// attached to a service account, such as when: +// - you are authenticated to the Storage Client with a token source, +// - you are using a custom HTTP client, or +// - you are logged into [gcloud using application default credentials] +// without [impersonation enabled]. +// To sign the URL with only the GoogleAccessID set you require: +// - the [IAM Service Account Credentials API enabled], and +// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. + +// [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication +// [Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth +// [gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login +// [impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account +// [IAM Service Account Credentials API enabled]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) { if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) { return SignedURL(b.name, object, opts) @@ -213,18 +229,34 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, // GenerateSignedPostPolicyV4 generates a PostPolicyV4 value from bucket, object and opts. // The generated URL and fields will then allow an unauthenticated client to perform multipart uploads. // -// This method only requires the Expires field in the specified PostPolicyV4Options -// to be non-nil. If not provided, it attempts to fill the GoogleAccessID and PrivateKey -// from the GOOGLE_APPLICATION_CREDENTIALS environment variable. -// If you are authenticating with a custom HTTP client, Service Account based -// auto-detection will be hindered. +// This method requires the Expires field in the specified PostPolicyV4Options +// to be non-nil. // -// If no private key is found, it attempts to use the GoogleAccessID to sign the URL. -// This requires the IAM Service Account Credentials API to be enabled -// (https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview) -// and iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. -// If you do not want these fields set for you, you may pass them in through opts or use -// GenerateSignedPostPolicyV4(bucket, name string, opts *PostPolicyV4Options) instead. +// If the GoogleAccessID and PrivateKey fields are not provided, they will be +// automatically detected when: +// - you are authenticated to the Storage Client with a service account's +// downloaded private key, either directly in code or by setting the +// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), +// - your application is running on Google Compute Engine (GCE), or +// - you are logged into [gcloud using application default credentials] +// with [impersonation enabled]. +// In some cases, you may not need to set PrivateKey but must set GoogleAccessID. +// GoogleAccessID should be set to a service account that will be used to attempt +// to sign the PostPolicyV4. This is true of cases where credentials are provided +// but not attached to a service account, such as when: +// - you are authenticated to the Storage Client with a token source, +// - you are using a custom HTTP client, or +// - you are logged into [gcloud using application default credentials] +// without [impersonation enabled]. +// To generate the PostPolicyV4 with only the GoogleAccessID set you require: +// - the [IAM Service Account Credentials API enabled], and +// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. + +// [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication +// [Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth +// [gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login +// [impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account +// [IAM Service Account Credentials API enabled]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) { if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) { return GenerateSignedPostPolicyV4(b.name, object, opts) From bdd9fb87b58981c90ee8ce87056dd91527236682 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Tue, 30 Aug 2022 15:33:41 -0500 Subject: [PATCH 4/8] revert unrelated changes --- storage/bucket.go | 63 +++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 30 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 72dc20ee9719..925a9b17939d 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -653,7 +653,12 @@ const ( // // All configured conditions must be met for the associated action to be taken. type LifecycleCondition struct { + // AllObjects is used to select all objects in a bucket by + // setting AgeInDays to 0. + AllObjects bool + // AgeInDays is the age of the object in days. + // If you want to set AgeInDays to `0` use AllObjects set to `true`. AgeInDays int64 // CreatedBefore is the time the object was created. @@ -671,10 +676,12 @@ type LifecycleCondition struct { // DaysSinceCustomTime is the days elapsed since the CustomTime date of the // object. This condition can only be satisfied if CustomTime has been set. + // Note: Using `0` as the value will be ignored by the library and not sent to the API. DaysSinceCustomTime int64 // DaysSinceNoncurrentTime is the days elapsed since the noncurrent timestamp // of the object. This condition is relevant only for versioned objects. + // Note: Using `0` as the value will be ignored by the library and not sent to the API. DaysSinceNoncurrentTime int64 // Liveness specifies the object's liveness. Relevant only for versioned objects @@ -706,6 +713,7 @@ type LifecycleCondition struct { // If the value is N, this condition is satisfied when there are at least N // versions (including the live version) newer than this version of the // object. + // Note: Using `0` as the value will be ignored by the library and not sent to the API. NumNewerVersions int64 } @@ -1464,19 +1472,6 @@ func toCORSFromProto(rc []*storagepb.Bucket_Cors) []CORS { return out } -// Used to handle breaking change in Autogen Storage client OLM Age field -// from int64 to *int64 gracefully in the manual client -// TODO(#6240): Method should be removed once breaking change is made and introduced to this client -func setAgeCondition(age int64, ageField interface{}) { - c := reflect.ValueOf(ageField).Elem() - switch c.Kind() { - case reflect.Int64: - c.SetInt(age) - case reflect.Ptr: - c.Set(reflect.ValueOf(&age)) - } -} - func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { var rl raw.BucketLifecycle if len(l.Rules) == 0 { @@ -1498,7 +1493,15 @@ func toRawLifecycle(l Lifecycle) *raw.BucketLifecycle { }, } - setAgeCondition(r.Condition.AgeInDays, &rr.Condition.Age) + // AllObjects takes precedent when both AllObjects and AgeInDays are set + // Rationale: If you've opted into using AllObjects, it makes sense that you + // understand the implications of how this option works with AgeInDays. + if r.Condition.AllObjects { + rr.Condition.Age = googleapi.Int64(0) + rr.Condition.ForceSendFields = []string{"Age"} + } else if r.Condition.AgeInDays > 0 { + rr.Condition.Age = googleapi.Int64(r.Condition.AgeInDays) + } switch r.Condition.Liveness { case LiveAndArchived: @@ -1547,6 +1550,11 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle { }, } + // TODO(#6205): This may not be needed for gRPC + if r.Condition.AllObjects { + rr.Condition.AgeDays = proto.Int32(0) + } + switch r.Condition.Liveness { case LiveAndArchived: rr.Condition.IsLive = nil @@ -1570,21 +1578,6 @@ func toProtoLifecycle(l Lifecycle) *storagepb.Bucket_Lifecycle { return &rl } -// Used to handle breaking change in Autogen Storage client OLM Age field -// from int64 to *int64 gracefully in the manual client -// TODO(#6240): Method should be removed once breaking change is made and introduced to this client -func getAgeCondition(ageField interface{}) int64 { - v := reflect.ValueOf(ageField) - if v.Kind() == reflect.Int64 { - return v.Interface().(int64) - } else if v.Kind() == reflect.Ptr { - if val, ok := v.Interface().(*int64); ok { - return *val - } - } - return 0 -} - func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { var l Lifecycle if rl == nil { @@ -1605,7 +1598,12 @@ func toLifecycle(rl *raw.BucketLifecycle) Lifecycle { NumNewerVersions: rr.Condition.NumNewerVersions, }, } - r.Condition.AgeInDays = getAgeCondition(rr.Condition.Age) + if rr.Condition.Age != nil { + r.Condition.AgeInDays = *rr.Condition.Age + if *rr.Condition.Age == 0 { + r.Condition.AllObjects = true + } + } if rr.Condition.IsLive == nil { r.Condition.Liveness = LiveAndArchived @@ -1651,6 +1649,11 @@ func toLifecycleFromProto(rl *storagepb.Bucket_Lifecycle) Lifecycle { }, } + // TODO(#6205): This may not be needed for gRPC + if rr.GetCondition().GetAgeDays() == 0 { + r.Condition.AllObjects = true + } + if rr.GetCondition().IsLive == nil { r.Condition.Liveness = LiveAndArchived } else if rr.GetCondition().GetIsLive() { From 4f8805956fcf39613405f413248711622eb9f2ad Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Tue, 30 Aug 2022 19:51:29 -0500 Subject: [PATCH 5/8] gofmt --- storage/bucket.go | 57 +++++++++++++++++++++++++---------------------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 925a9b17939d..75586fea39cf 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -169,24 +169,26 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) ( // // If the GoogleAccessID and PrivateKey fields are not provided, they will be // automatically detected when: -// - you are authenticated to the Storage Client with a service account's -// downloaded private key, either directly in code or by setting the -// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), -// - your application is running on Google Compute Engine (GCE), or -// - you are logged into [gcloud using application default credentials] -// with [impersonation enabled]. +// - you are authenticated to the Storage Client with a service account's +// downloaded private key, either directly in code or by setting the +// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), +// - your application is running on Google Compute Engine (GCE), or +// - you are logged into [gcloud using application default credentials] +// with [impersonation enabled]. +// // In some cases, you may not need to set PrivateKey but must set GoogleAccessID. // GoogleAccessID should be set to a service account that will be used to attempt // to sign the URL. This is true of cases where credentials are provided but not // attached to a service account, such as when: -// - you are authenticated to the Storage Client with a token source, -// - you are using a custom HTTP client, or -// - you are logged into [gcloud using application default credentials] -// without [impersonation enabled]. +// - you are authenticated to the Storage Client with a token source, +// - you are using a custom HTTP client, or +// - you are logged into [gcloud using application default credentials] +// without [impersonation enabled]. +// // To sign the URL with only the GoogleAccessID set you require: -// - the [IAM Service Account Credentials API enabled], and -// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. - +// - the [IAM Service Account Credentials API enabled], and +// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. +// // [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication // [Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth // [gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login @@ -234,25 +236,26 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, // // If the GoogleAccessID and PrivateKey fields are not provided, they will be // automatically detected when: -// - you are authenticated to the Storage Client with a service account's -// downloaded private key, either directly in code or by setting the -// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), -// - your application is running on Google Compute Engine (GCE), or -// - you are logged into [gcloud using application default credentials] -// with [impersonation enabled]. +// - you are authenticated to the Storage Client with a service account's +// downloaded private key, either directly in code or by setting the +// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), +// - your application is running on Google Compute Engine (GCE), or +// - you are logged into [gcloud using application default credentials] +// with [impersonation enabled]. +// // In some cases, you may not need to set PrivateKey but must set GoogleAccessID. // GoogleAccessID should be set to a service account that will be used to attempt // to sign the PostPolicyV4. This is true of cases where credentials are provided // but not attached to a service account, such as when: -// - you are authenticated to the Storage Client with a token source, -// - you are using a custom HTTP client, or -// - you are logged into [gcloud using application default credentials] -// without [impersonation enabled]. +// - you are authenticated to the Storage Client with a token source, +// - you are using a custom HTTP client, or +// - you are logged into [gcloud using application default credentials] +// without [impersonation enabled]. +// // To generate the PostPolicyV4 with only the GoogleAccessID set you require: -// - the [IAM Service Account Credentials API enabled], and -// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. - -// [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication +// - the [IAM Service Account Credentials API enabled], and +// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. +// // [Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth // [gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login // [impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account From 7aa67de9745f018d70dd98c8a6d39fe146f2d4cf Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Tue, 6 Sep 2022 16:45:46 -0500 Subject: [PATCH 6/8] suggestions --- storage/bucket.go | 65 ++++++------------------------------------ storage/bucket_test.go | 7 +++++ storage/doc.go | 31 ++++++++++++++++++-- 3 files changed, 44 insertions(+), 59 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 75586fea39cf..43fadd402527 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -165,35 +165,12 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) ( // For more information about signed URLs, see "[Overview of access control]." // // This method requires the Method and Expires fields in the specified -// SignedURLOptions to be non-nil. -// -// If the GoogleAccessID and PrivateKey fields are not provided, they will be -// automatically detected when: -// - you are authenticated to the Storage Client with a service account's -// downloaded private key, either directly in code or by setting the -// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), -// - your application is running on Google Compute Engine (GCE), or -// - you are logged into [gcloud using application default credentials] -// with [impersonation enabled]. -// -// In some cases, you may not need to set PrivateKey but must set GoogleAccessID. -// GoogleAccessID should be set to a service account that will be used to attempt -// to sign the URL. This is true of cases where credentials are provided but not -// attached to a service account, such as when: -// - you are authenticated to the Storage Client with a token source, -// - you are using a custom HTTP client, or -// - you are logged into [gcloud using application default credentials] -// without [impersonation enabled]. -// -// To sign the URL with only the GoogleAccessID set you require: -// - the [IAM Service Account Credentials API enabled], and -// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. +// SignedURLOptions to be non-nil. You may need to set the GoogleAccessID and +// PrivateKey fields in some cases. Read more on the [automatic detection of credentials] +// for this method. // // [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication -// [Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth -// [gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login -// [impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account -// [IAM Service Account Credentials API enabled]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview +// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4] func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) { if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) { return SignedURL(b.name, object, opts) @@ -232,34 +209,10 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, // The generated URL and fields will then allow an unauthenticated client to perform multipart uploads. // // This method requires the Expires field in the specified PostPolicyV4Options -// to be non-nil. -// -// If the GoogleAccessID and PrivateKey fields are not provided, they will be -// automatically detected when: -// - you are authenticated to the Storage Client with a service account's -// downloaded private key, either directly in code or by setting the -// GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), -// - your application is running on Google Compute Engine (GCE), or -// - you are logged into [gcloud using application default credentials] -// with [impersonation enabled]. -// -// In some cases, you may not need to set PrivateKey but must set GoogleAccessID. -// GoogleAccessID should be set to a service account that will be used to attempt -// to sign the PostPolicyV4. This is true of cases where credentials are provided -// but not attached to a service account, such as when: -// - you are authenticated to the Storage Client with a token source, -// - you are using a custom HTTP client, or -// - you are logged into [gcloud using application default credentials] -// without [impersonation enabled]. -// -// To generate the PostPolicyV4 with only the GoogleAccessID set you require: -// - the [IAM Service Account Credentials API enabled], and -// - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. +// to be non-nil. You may need to set the GoogleAccessID and PrivateKey fields +// in some cases. Read more on the [automatic detection of credentials] for this method. // -// [Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth -// [gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login -// [impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account -// [IAM Service Account Credentials API enabled]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview +// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4] func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) { if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) { return GenerateSignedPostPolicyV4(b.name, object, opts) @@ -318,7 +271,7 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { } else if sa.CredType == "service_account" && sa.ClientEmail != "" { return sa.ClientEmail, nil } else { - returnErr = errors.New("unable to parse credentials") + returnErr = errors.New("unable to parse credentials; only service_account and impersonated_service_account credentials are supported") } } @@ -334,7 +287,7 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { } } - return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %v", returnErr) + return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4]).", returnErr) } func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) { diff --git a/storage/bucket_test.go b/storage/bucket_test.go index b8cc20c96a59..7a53625d5f01 100644 --- a/storage/bucket_test.go +++ b/storage/bucket_test.go @@ -843,6 +843,13 @@ func TestDetectDefaultGoogleAccessID(t *testing.T) { }, expectSuccess: true, }, + { + name: "no creds", + creds: func(_ string) string { + return "" + }, + expectSuccess: false, + }, } for _, tc := range testCases { diff --git a/storage/doc.go b/storage/doc.go index cce44c4365ea..2b8ed2d89a01 100644 --- a/storage/doc.go +++ b/storage/doc.go @@ -214,8 +214,8 @@ since you read it. Here is how to express that: You can obtain a URL that lets anyone read or write an object for a limited time. Signing a URL requires credentials authorized to sign a URL. To use the same -authentication that was used when instantiating the Storage client, use the -BucketHandle.SignedURL method. +authentication that was used when instantiating the Storage client, use +[BucketHandle.SignedURL]. url, err := client.Bucket(bucketName).SignedURL(objectName, opts) if err != nil { @@ -224,7 +224,7 @@ BucketHandle.SignedURL method. fmt.Println(url) You can also sign a URL wihout creating a client. See the documentation of -SignedURL for details. +[SignedURL] for details. url, err := storage.SignedURL(bucketName, "shared-object", opts) if err != nil { @@ -232,6 +232,26 @@ SignedURL for details. } fmt.Println(url) +# Credential detection for [BucketHandle.SignedURL] and [BucketHandle.GenerateSignedPostPolicyV4] + +If the GoogleAccessID and PrivateKey option fields are not provided, they will +be automatically detected if any of the following are true: + - you are authenticated to the Storage Client with a service account's + downloaded private key, either directly in code or by setting the + GOOGLE_APPLICATION_CREDENTIALS environment variable (see [Other Environments]), + - your application is running on Google Compute Engine (GCE), or + - you are logged into [gcloud using application default credentials] + with [impersonation enabled]. + +Detecting GoogleAccessID may not be possible if you are authenticated using a +token source or using [option.WithHTTPClient]. In this case, you can provide a +service account email for GoogleAccessID and the client will attempt to sign +the URL or Post Policy using that service account. + +To sign without a custom signing function you require: + - the [IAM Service Account Credentials API] enabled, and + - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. + # Post Policy V4 Signed Request A type of signed request that allows uploads through HTML forms directly to Cloud Storage with @@ -296,5 +316,10 @@ client (using Client.SetRetry). For example: if err := o.Delete(ctx); err != nil { // Handle err. } + +[Other Environments]: https://cloud.google.com/storage/docs/authentication#libauth +[gcloud using application default credentials]: https://cloud.google.com/sdk/gcloud/reference/auth/application-default/login +[impersonation enabled]: https://cloud.google.com/sdk/gcloud/reference#--impersonate-service-account +[IAM Service Account Credentials API]: https://console.developers.google.com/apis/api/iamcredentials.googleapis.com/overview */ package storage // import "cloud.google.com/go/storage" From 5eddd89b36b097d6e8432afe89002c01257753a1 Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Wed, 7 Sep 2022 14:27:22 -0500 Subject: [PATCH 7/8] fix lint --- storage/bucket.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/bucket.go b/storage/bucket.go index 43fadd402527..17d8ef437895 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -287,7 +287,7 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { } } - return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4]).", returnErr) + return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4])", returnErr) } func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) { From 6fc44ef9a5f173f9d4d6c9999076a42ebe56d9be Mon Sep 17 00:00:00 2001 From: BrennaEpp Date: Wed, 14 Sep 2022 14:00:59 -0700 Subject: [PATCH 8/8] suggestions --- storage/bucket.go | 6 +++--- storage/doc.go | 38 +++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/storage/bucket.go b/storage/bucket.go index 17d8ef437895..20302d1625e7 100644 --- a/storage/bucket.go +++ b/storage/bucket.go @@ -170,7 +170,7 @@ func (b *BucketHandle) Update(ctx context.Context, uattrs BucketAttrsToUpdate) ( // for this method. // // [Overview of access control]: https://cloud.google.com/storage/docs/accesscontrol#signed_urls_query_string_authentication -// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4] +// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4] func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, error) { if opts.GoogleAccessID != "" && (opts.SignBytes != nil || len(opts.PrivateKey) > 0) { return SignedURL(b.name, object, opts) @@ -212,7 +212,7 @@ func (b *BucketHandle) SignedURL(object string, opts *SignedURLOptions) (string, // to be non-nil. You may need to set the GoogleAccessID and PrivateKey fields // in some cases. Read more on the [automatic detection of credentials] for this method. // -// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4] +// [automatic detection of credentials]: https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4] func (b *BucketHandle) GenerateSignedPostPolicyV4(object string, opts *PostPolicyV4Options) (*PostPolicyV4, error) { if opts.GoogleAccessID != "" && (opts.SignRawBytes != nil || opts.SignBytes != nil || len(opts.PrivateKey) > 0) { return GenerateSignedPostPolicyV4(b.name, object, opts) @@ -287,7 +287,7 @@ func (b *BucketHandle) detectDefaultGoogleAccessID() (string, error) { } } - return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_detection_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4])", returnErr) + return "", fmt.Errorf("storage: unable to detect default GoogleAccessID: %w. Please provide the GoogleAccessID or use a supported means for autodetecting it (see https://pkg.go.dev/cloud.google.com/go/storage#hdr-Credential_requirements_for_[BucketHandle.SignedURL]_and_[BucketHandle.GenerateSignedPostPolicyV4])", returnErr) } func (b *BucketHandle) defaultSignBytesFunc(email string) func([]byte) ([]byte, error) { diff --git a/storage/doc.go b/storage/doc.go index 2b8ed2d89a01..dbd3ba512d79 100644 --- a/storage/doc.go +++ b/storage/doc.go @@ -232,7 +232,22 @@ You can also sign a URL wihout creating a client. See the documentation of } fmt.Println(url) -# Credential detection for [BucketHandle.SignedURL] and [BucketHandle.GenerateSignedPostPolicyV4] +# Post Policy V4 Signed Request + +A type of signed request that allows uploads through HTML forms directly to Cloud Storage with +temporary permission. Conditions can be applied to restrict how the HTML form is used and exercised +by a user. + +For more information, please see https://cloud.google.com/storage/docs/xml-api/post-object as well +as the documentation of BucketHandle.GenerateSignedPostPolicyV4. + + pv4, err := client.Bucket(bucketName).GenerateSignedPostPolicyV4(objectName, opts) + if err != nil { + // TODO: Handle error. + } + fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields) + +# Credential requirements for [BucketHandle.SignedURL] and [BucketHandle.GenerateSignedPostPolicyV4] If the GoogleAccessID and PrivateKey option fields are not provided, they will be automatically detected if any of the following are true: @@ -248,24 +263,9 @@ token source or using [option.WithHTTPClient]. In this case, you can provide a service account email for GoogleAccessID and the client will attempt to sign the URL or Post Policy using that service account. -To sign without a custom signing function you require: - - the [IAM Service Account Credentials API] enabled, and - - iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account. - -# Post Policy V4 Signed Request - -A type of signed request that allows uploads through HTML forms directly to Cloud Storage with -temporary permission. Conditions can be applied to restrict how the HTML form is used and exercised -by a user. - -For more information, please see https://cloud.google.com/storage/docs/xml-api/post-object as well -as the documentation of BucketHandle.GenerateSignedPostPolicyV4. - - pv4, err := client.Bucket(bucketName).GenerateSignedPostPolicyV4(objectName, opts) - if err != nil { - // TODO: Handle error. - } - fmt.Printf("URL: %s\nFields; %v\n", pv4.URL, pv4.Fields) +To generate the signature, you must have: +- iam.serviceAccounts.signBlob permissions on the GoogleAccessID service account, and +- the [IAM Service Account Credentials API] enabled (unless authenticating with a downloaded private key). # Errors