diff --git a/google/cloud/storage/client.cc b/google/cloud/storage/client.cc index 8727768182b5..25e4271f8a35 100644 --- a/google/cloud/storage/client.cc +++ b/google/cloud/storage/client.cc @@ -23,6 +23,7 @@ #include "google/cloud/internal/make_status.h" #include "google/cloud/internal/opentelemetry.h" #include "google/cloud/log.h" +#include "absl/strings/str_split.h" #include #include #include @@ -389,7 +390,7 @@ StatusOr Client::SignUrlV2( std::string signature = curl.MakeEscapedString(encoded).get(); std::ostringstream os; - os << "https://storage.googleapis.com/" << request.bucket_name(); + os << Endpoint() << '/' << request.bucket_name(); if (!request.object_name().empty()) { os << '/' << curl.MakeEscapedString(request.object_name()).get(); } @@ -481,6 +482,21 @@ std::string CreateRandomPrefixName(std::string const& prefix) { "abcdefghijklmnopqrstuvwxyz"); } +std::string Client::Endpoint() const { + return connection_->options().get(); +} + +// This can be optimized to not have a lot of string copies. +// But the code is rarely used and not in any critical path. +std::string Client::EndpointAuthority() const { + auto endpoint = Endpoint(); + auto endpoint_authority = absl::string_view(endpoint); + if (!absl::ConsumePrefix(&endpoint_authority, "https://")) { + absl::ConsumePrefix(&endpoint_authority, "http://"); + } + return std::string(endpoint_authority); +} + namespace internal { Client ClientImplDetails::CreateWithDecorations( diff --git a/google/cloud/storage/client.h b/google/cloud/storage/client.h index 01f9cb79a547..3934938e46f0 100644 --- a/google/cloud/storage/client.h +++ b/google/cloud/storage/client.h @@ -2996,7 +2996,8 @@ class Client { google::cloud::internal::OptionsSpan const span( SpanOptions(std::forward(options)...)); internal::V4SignUrlRequest request(std::move(verb), std::move(bucket_name), - std::move(object_name)); + std::move(object_name), + EndpointAuthority()); request.set_multiple_options(std::forward(options)...); return SignUrlV4(std::move(request)); } @@ -3082,7 +3083,8 @@ class Client { PolicyDocumentV4 document, Options&&... options) { google::cloud::internal::OptionsSpan const span( SpanOptions(std::forward(options)...)); - internal::PolicyDocumentV4Request request(std::move(document)); + internal::PolicyDocumentV4Request request(std::move(document), + EndpointAuthority()); request.set_multiple_options(std::forward(options)...); return SignPolicyDocumentV4(std::move(request)); } @@ -3492,6 +3494,12 @@ class Client { StatusOr SignPolicyDocumentV4( internal::PolicyDocumentV4Request request); + // The configured endpoint, including any scheme and port. + std::string Endpoint() const; + + // The hostname:port part of the configured endpoint. + std::string EndpointAuthority() const; + std::shared_ptr connection_; }; diff --git a/google/cloud/storage/client_sign_policy_document_test.cc b/google/cloud/storage/client_sign_policy_document_test.cc index af20656d94bf..e70e585544d6 100644 --- a/google/cloud/storage/client_sign_policy_document_test.cc +++ b/google/cloud/storage/client_sign_policy_document_test.cc @@ -33,6 +33,7 @@ using ::google::cloud::internal::CurrentOptions; using ::google::cloud::storage::testing::canonical_errors::TransientError; using ::testing::HasSubstr; using ::testing::Return; +using ::testing::StartsWith; constexpr char kJsonKeyfileContents[] = R"""({ "type": "service_account", @@ -58,9 +59,12 @@ std::string Dec64(std::string const& s) { return std::string(res.begin(), res.end()); }; -Client CreateClientForTest() { - return Client(Options{}.set( - MakeServiceAccountCredentials(kJsonKeyfileContents))); +Client CreateClientForTest( + std::string endpoint = "https://storage.googleapis.com") { + return Client(Options{} + .set( + MakeServiceAccountCredentials(kJsonKeyfileContents)) + .set(std::move(endpoint))); } /** @@ -255,6 +259,16 @@ TEST(CreateSignedPolicyDocTest, SignV4VirtualHostname) { EXPECT_EQ("https://test-bucket.storage.googleapis.com/", actual->url); } +TEST(CreateSignedPolicyDocTest, SignV4CustomEndpoint) { + auto const custom_endpoint = std::string{"https://storage.mydomain.com"}; + auto client = CreateClientForTest(custom_endpoint); + auto actual = + client.GenerateSignedPostPolicyV4(CreatePolicyDocumentV4ForTest()); + + ASSERT_STATUS_OK(actual); + EXPECT_THAT(actual->url, StartsWith(custom_endpoint)); +} + } // namespace GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage diff --git a/google/cloud/storage/client_sign_url_test.cc b/google/cloud/storage/client_sign_url_test.cc index 680642ccb73c..8e68a9edeedb 100644 --- a/google/cloud/storage/client_sign_url_test.cc +++ b/google/cloud/storage/client_sign_url_test.cc @@ -18,6 +18,7 @@ #include "google/cloud/storage/testing/canonical_errors.h" #include "google/cloud/storage/testing/client_unit_test.h" #include "google/cloud/testing_util/status_matchers.h" +#include "google/cloud/universe_domain_options.h" #include namespace google { @@ -28,9 +29,11 @@ namespace { using ::google::cloud::internal::CurrentOptions; using ::google::cloud::storage::testing::canonical_errors::TransientError; +using ::google::cloud::testing_util::IsOkAndHolds; using ::google::cloud::testing_util::StatusIs; using ::testing::HasSubstr; using ::testing::Return; +using ::testing::StartsWith; constexpr char kJsonKeyfileContents[] = R"""({ "type": "service_account", @@ -107,6 +110,36 @@ TEST_F(CreateSignedUrlTest, V2SignRemote) { EXPECT_THAT(*actual, HasSubstr(expected_signed_blob_safe)); } +/// @test Verify that CreateV2SignedUrl() respects the custom endpoint. +TEST_F(CreateSignedUrlTest, V2SignCustomEndpoint) { + auto const custom_endpoint = std::string{"https://storage.mydomain.com"}; + + Options options = Options{} + .set( + MakeServiceAccountCredentials(kJsonKeyfileContents)) + .set(custom_endpoint); + Client client(options); + StatusOr actual = + client.CreateV2SignedUrl("GET", "test-bucket", "test-object"); + EXPECT_THAT(actual, IsOkAndHolds(StartsWith(custom_endpoint))); +} + +/// @test Verify that CreateV2SignedUrl() respects the custom universe domain. +TEST_F(CreateSignedUrlTest, V2SignCustomUniverseDomain) { + auto const custom_ud = std::string{"mydomain.com"}; + + Options options = + Options{} + .set( + MakeServiceAccountCredentials(kJsonKeyfileContents)) + .set(custom_ud); + Client client(options); + + StatusOr actual = + client.CreateV2SignedUrl("GET", "test-bucket", "test-object"); + EXPECT_THAT(actual, IsOkAndHolds(HasSubstr(custom_ud))); +} + // This is a placeholder service account JSON file that is inactive. It's fine // for it to be public. constexpr char kJsonKeyfileContentsForV4[] = R"""({ @@ -231,6 +264,49 @@ TEST_F(CreateSignedUrlTest, V4SignRemote) { EXPECT_THAT(*actual, HasSubstr(expected_signed_blob_hex)); } +/// @test Verify that CreateV4SignedUrl() respects the custom endpoint. +TEST_F(CreateSignedUrlTest, V4SignCustomEndpoint) { + auto const custom_endpoint = std::string{"https://storage.mydomain.com"}; + std::string const bucket_name = "test-bucket"; + std::string const object_name = "test-object"; + std::string const date = "2019-02-01T09:00:00Z"; + auto const valid_for = std::chrono::seconds(10); + + Options options = + Options{} + .set( + MakeServiceAccountCredentials(kJsonKeyfileContentsForV4)) + .set(custom_endpoint); + Client client(options); + + auto actual = client.CreateV4SignedUrl( + "GET", bucket_name, object_name, + SignedUrlTimestamp(google::cloud::internal::ParseRfc3339(date).value()), + SignedUrlDuration(valid_for)); + EXPECT_THAT(actual, IsOkAndHolds(StartsWith(custom_endpoint))); +} + +/// @test Verify that CreateV4SignUrl() respects the custom universe domain. +TEST_F(CreateSignedUrlTest, V4SignCustomUniverseDomain) { + auto const custom_ud = std::string{"mydomain.com"}; + std::string const bucket_name = "test-bucket"; + std::string const object_name = "test-object"; + std::string const date = "2019-02-01T09:00:00Z"; + auto const valid_for = std::chrono::seconds(10); + + Options options = + Options{} + .set( + MakeServiceAccountCredentials(kJsonKeyfileContentsForV4)) + .set(custom_ud); + Client client(options); + auto actual = client.CreateV4SignedUrl( + "GET", bucket_name, object_name, + SignedUrlTimestamp(google::cloud::internal::ParseRfc3339(date).value()), + SignedUrlDuration(valid_for)); + EXPECT_THAT(actual, IsOkAndHolds(HasSubstr(custom_ud))); +} + TEST_F(CreateSignedUrlTest, V4SignRemoteNoSigningEmail) { EXPECT_CALL(*mock_, SignBlob).Times(0); auto client = ClientForMock(); diff --git a/google/cloud/storage/internal/policy_document_request.cc b/google/cloud/storage/internal/policy_document_request.cc index 4cbbab5e5e59..c077c916e95e 100644 --- a/google/cloud/storage/internal/policy_document_request.cc +++ b/google/cloud/storage/internal/policy_document_request.cc @@ -275,11 +275,11 @@ std::string PolicyDocumentV4Request::Url() const { return scheme_ + "://" + *bucket_bound_domain_ + "/"; } if (virtual_host_name_) { - return scheme_ + "://" + policy_document().bucket + - ".storage.googleapis.com/"; + return scheme_ + "://" + policy_document().bucket + "." + + endpoint_authority_ + "/"; } - return scheme_ + "://storage.googleapis.com/" + policy_document().bucket + - "/"; + return scheme_ + "://" + endpoint_authority_ + "/" + + policy_document().bucket + "/"; } std::string PolicyDocumentV4Request::Credentials() const { @@ -342,7 +342,8 @@ std::map PolicyDocumentV4Request::RequiredFormFields() } std::ostream& operator<<(std::ostream& os, PolicyDocumentV4Request const& r) { - return os << "PolicyDocumentRequest={" << r.StringToSign() << "}"; + return os << "PolicyDocumentRequest={endpoint_authority=" + << r.endpoint_authority() << "," << r.StringToSign() << "}"; } } // namespace internal diff --git a/google/cloud/storage/internal/policy_document_request.h b/google/cloud/storage/internal/policy_document_request.h index d82649789539..c752518f16aa 100644 --- a/google/cloud/storage/internal/policy_document_request.h +++ b/google/cloud/storage/internal/policy_document_request.h @@ -104,7 +104,15 @@ std::ostream& operator<<(std::ostream& os, PolicyDocumentRequest const& r); class PolicyDocumentV4Request { public: - PolicyDocumentV4Request() : scheme_("https") {} + PolicyDocumentV4Request() + : PolicyDocumentV4Request("storage.googleapis.com") {} + explicit PolicyDocumentV4Request(std::string endpoint_authority) + : scheme_("https"), endpoint_authority_(std::move(endpoint_authority)) {} + PolicyDocumentV4Request(PolicyDocumentV4 document, + std::string endpoint_authority) + : PolicyDocumentV4Request(std::move(endpoint_authority)) { + document_ = std::move(document); + } explicit PolicyDocumentV4Request(PolicyDocumentV4 document) : PolicyDocumentV4Request() { document_ = std::move(document); @@ -125,6 +133,8 @@ class PolicyDocumentV4Request { return signing_account_delegates_; } + std::string endpoint_authority() const { return endpoint_authority_; } + void SetOption(SigningAccount const& o) { signing_account_ = o; } void SetOption(SigningAccountDelegates const& o) { @@ -191,6 +201,7 @@ class PolicyDocumentV4Request { absl::optional bucket_bound_domain_; std::string scheme_; bool virtual_host_name_{false}; + std::string endpoint_authority_; }; std::ostream& operator<<(std::ostream& os, PolicyDocumentV4Request const& r); diff --git a/google/cloud/storage/internal/policy_document_request_test.cc b/google/cloud/storage/internal/policy_document_request_test.cc index defe689cc814..ad04ee24fd46 100644 --- a/google/cloud/storage/internal/policy_document_request_test.cc +++ b/google/cloud/storage/internal/policy_document_request_test.cc @@ -28,6 +28,7 @@ namespace { using ::google::cloud::testing_util::IsOkAndHolds; using ::google::cloud::testing_util::StatusIs; using ::testing::ElementsAre; +using ::testing::StartsWith; TEST(PolicyDocumentRequest, SigningAccount) { PolicyDocumentRequest request; @@ -119,7 +120,8 @@ TEST(PolicyDocumentV4Request, Printing) { std::stringstream stream; stream << req; EXPECT_EQ( - "PolicyDocumentRequest={{\"conditions\":[{\"bucket\":\"test-bucket\"},{" + "PolicyDocumentRequest={endpoint_authority=storage.googleapis.com," + "{\"conditions\":[{\"bucket\":\"test-bucket\"},{" "\"key\":\"test-object\"},{\"x-goog-date\":\"20100616T111111Z\"},{\"x-" "goog-credential\":\"/20100616/auto/storage/" "goog4_request\"},{\"x-goog-algorithm\":\"GOOG4-RSA-SHA256\"}]," @@ -153,6 +155,27 @@ TEST(PolicyDocumentV4Request, RequiredFormFields) { EXPECT_EQ(expected_fields, req.RequiredFormFields()); } +TEST(PolicyDocumentV4Request, Url) { + PolicyDocumentV4 doc; + doc.bucket = "test-bucket"; + auto const custom_endpoint_authority = std::string{"storage.mydomain.com"}; + PolicyDocumentV4Request request(doc, custom_endpoint_authority); + EXPECT_THAT(request.Url(), + StartsWith("https://" + custom_endpoint_authority)); +} + +TEST(PolicyDocumentV4Request, UrlWithVirtualHostName) { + PolicyDocumentV4 doc; + doc.bucket = "test-bucket"; + auto const custom_endpoint_authority = std::string{"storage.mydomain.com"}; + PolicyDocumentV4Request request(doc, custom_endpoint_authority); + request.SetOption(VirtualHostname(true)); + + auto const expected_url = std::string{"https://" + doc.bucket + "." + + custom_endpoint_authority + "/"}; + EXPECT_EQ(request.Url(), expected_url); +} + } // namespace } // namespace internal GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END diff --git a/google/cloud/storage/internal/signed_url_requests.cc b/google/cloud/storage/internal/signed_url_requests.cc index e923fb414945..43b4406c4cfb 100644 --- a/google/cloud/storage/internal/signed_url_requests.cc +++ b/google/cloud/storage/internal/signed_url_requests.cc @@ -100,7 +100,8 @@ std::string V2SignUrlRequest::StringToSign() const { } std::ostream& operator<<(std::ostream& os, V2SignUrlRequest const& r) { - return os << "SingUrlRequest={" << r.StringToSign() << "}"; + return os << "SingUrlRequest={" << r.endpoint_authority() << "," + << r.StringToSign() << "}"; } namespace { @@ -235,13 +236,14 @@ Status V4SignUrlRequest::Validate() { } std::string V4SignUrlRequest::Hostname() { + auto endpoint_authority = common_request_.endpoint_authority(); if (virtual_host_name_) { - return common_request_.bucket_name() + ".storage.googleapis.com"; + return common_request_.bucket_name() + "." + endpoint_authority; } if (domain_named_bucket_) { return *domain_named_bucket_; } - return "storage.googleapis.com"; + return endpoint_authority; } std::string V4SignUrlRequest::HostnameWithBucket() { @@ -318,7 +320,7 @@ std::string V4SignUrlRequest::PayloadHashValue() const { } std::ostream& operator<<(std::ostream& os, V4SignUrlRequest const& r) { - return os << "V4SignUrlRequest={" + return os << "V4SignUrlRequest={" << r.endpoint_authority() << "," << r.CanonicalRequest("placeholder-client-id") << "," << r.StringToSign("placeholder-client-id") << "}"; } diff --git a/google/cloud/storage/internal/signed_url_requests.h b/google/cloud/storage/internal/signed_url_requests.h index 6fd63111425d..2df11421a4ef 100644 --- a/google/cloud/storage/internal/signed_url_requests.h +++ b/google/cloud/storage/internal/signed_url_requests.h @@ -38,14 +38,16 @@ class SignUrlRequestCommon { public: SignUrlRequestCommon() = default; SignUrlRequestCommon(std::string verb, std::string bucket_name, - std::string object_name) + std::string object_name, std::string endpoint_authority) : verb_(std::move(verb)), bucket_name_(std::move(bucket_name)), - object_name_(std::move(object_name)) {} + object_name_(std::move(object_name)), + endpoint_authority_(std::move(endpoint_authority)) {} std::string const& verb() const { return verb_; } std::string const& bucket_name() const { return bucket_name_; } std::string const& object_name() const { return object_name_; } + std::string const& endpoint_authority() const { return endpoint_authority_; } std::string const& sub_resource() const { return sub_resource_; } std::map const& extension_headers() const { return extension_headers_; @@ -94,6 +96,7 @@ class SignUrlRequestCommon { std::string verb_; std::string bucket_name_; std::string object_name_; + std::string endpoint_authority_; std::string sub_resource_; std::map extension_headers_; std::multimap query_parameters_; @@ -108,10 +111,11 @@ class SignUrlRequestCommon { class V2SignUrlRequest { public: V2SignUrlRequest() = default; - explicit V2SignUrlRequest(std::string verb, std::string bucket_name, - std::string object_name) + explicit V2SignUrlRequest( + std::string verb, std::string bucket_name, std::string object_name, + std::string endpoint_authority = "storage.googleapis.com") : common_request_(std::move(verb), std::move(bucket_name), - std::move(object_name)), + std::move(object_name), std::move(endpoint_authority)), expiration_time_(DefaultExpirationTime()) {} std::string const& verb() const { return common_request_.verb(); } @@ -121,6 +125,9 @@ class V2SignUrlRequest { std::string const& object_name() const { return common_request_.object_name(); } + std::string const& endpoint_authority() const { + return common_request_.endpoint_authority(); + } std::string const& sub_resource() const { return common_request_.sub_resource(); } @@ -220,10 +227,11 @@ std::ostream& operator<<(std::ostream& os, V2SignUrlRequest const& r); class V4SignUrlRequest { public: V4SignUrlRequest() : expires_(0) {} - explicit V4SignUrlRequest(std::string verb, std::string bucket_name, - std::string object_name) + explicit V4SignUrlRequest( + std::string verb, std::string bucket_name, std::string object_name, + std::string endpoint_authority = "storage.googleapis.com") : common_request_(std::move(verb), std::move(bucket_name), - std::move(object_name)), + std::move(object_name), std::move(endpoint_authority)), scheme_("https"), timestamp_(DefaultTimestamp()), expires_(DefaultExpires()), @@ -236,6 +244,9 @@ class V4SignUrlRequest { std::string const& object_name() const { return common_request_.object_name(); } + std::string const& endpoint_authority() const { + return common_request_.endpoint_authority(); + } std::vector ObjectNameParts() const; diff --git a/google/cloud/storage/internal/signed_url_requests_test.cc b/google/cloud/storage/internal/signed_url_requests_test.cc index 8687e29d2952..22b04c822efb 100644 --- a/google/cloud/storage/internal/signed_url_requests_test.cc +++ b/google/cloud/storage/internal/signed_url_requests_test.cc @@ -28,6 +28,7 @@ namespace { using ::google::cloud::testing_util::StatusIs; using ::testing::ElementsAre; using ::testing::HasSubstr; +using ::testing::StartsWith; TEST(V2SignedUrlRequests, Sign) { V2SignUrlRequest request("GET", "test-bucket", "test-object"); @@ -641,6 +642,14 @@ TEST(V4SignedUrlRequests, BucketBoundHostnameReset) { EXPECT_EQ(expected, actual); } +TEST(V4SignedUrlRequests, CustomEndpoint) { + auto const custom_endpoint_authority = std::string{"storage.mydomain.com"}; + V4SignUrlRequest request("GET", "test-bucket", "test-object", + custom_endpoint_authority); + ASSERT_STATUS_OK(request.Validate()); + EXPECT_THAT(request.Hostname(), StartsWith(custom_endpoint_authority)); +} + TEST(DefaultCtorsWork, Trivial) { EXPECT_FALSE(ExpirationTime().has_value()); EXPECT_FALSE(AddExtensionHeaderOption().has_value()); diff --git a/google/cloud/storage/tests/signed_url_conformance_test.cc b/google/cloud/storage/tests/signed_url_conformance_test.cc index 8df28bfe4381..eb9371030795 100644 --- a/google/cloud/storage/tests/signed_url_conformance_test.cc +++ b/google/cloud/storage/tests/signed_url_conformance_test.cc @@ -23,6 +23,7 @@ #include "google/cloud/internal/getenv.h" #include "google/cloud/internal/time_utils.h" #include "google/cloud/terminate_handler.h" +#include "google/cloud/testing_util/scoped_environment.h" #include "google/cloud/testing_util/status_matchers.h" #include #include @@ -75,6 +76,8 @@ class V4SignedUrlConformanceTest class V4PostPolicyConformanceTest : public V4SignedUrlConformanceTest {}; TEST_P(V4SignedUrlConformanceTest, V4SignJson) { + testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT", + absl::nullopt); auto creds = oauth2::CreateServiceAccountCredentialsFromJsonFilePath( service_account_key_filename_); ASSERT_STATUS_OK(creds); @@ -176,6 +179,8 @@ INSTANTIATE_TEST_SUITE_P( }())); TEST_P(V4PostPolicyConformanceTest, V4PostPolicy) { + testing_util::ScopedEnvironment endpoint("CLOUD_STORAGE_EMULATOR_ENDPOINT", + absl::nullopt); auto creds = oauth2::CreateServiceAccountCredentialsFromJsonFilePath( service_account_key_filename_); ASSERT_STATUS_OK(creds);