From d9f071dd499e62dd87bba7b34f569bca822018ae Mon Sep 17 00:00:00 2001 From: Leonid Baraz Date: Fri, 17 Mar 2023 19:02:57 +0000 Subject: [PATCH] Implement event-based file upload Delegate implementation and unittests are provided for ChromeOS Ash only. Currently metadata and access are temporary placeholders, need to be finalized. Manually tested on VM. Positive unit tests coverage is complete, negative tests are provided for initialization, need to be added for next-step and finalization. Bug: b:264399295 Change-Id: I9ba990a09aec125d725eba4233f1061767701935 Reviewed-on: https://chromium-review.googlesource.com/c/chromium/src/+/4336709 Reviewed-by: Vignesh Shenvi Commit-Queue: Leonid Baraz Reviewed-by: Nicolas Ouellet-Payeur Auto-Submit: Leonid Baraz Cr-Commit-Position: refs/heads/main@{#1118797} --- chrome/browser/BUILD.gn | 4 +- .../upload/file_upload_impl.cc | 953 +++++++++++++++++- .../messaging_layer/upload/file_upload_impl.h | 98 +- .../upload/file_upload_impl_unittest.cc | 611 ++++++++++- .../messaging_layer/upload/file_upload_job.h | 2 +- .../messaging_layer/upload/upload_client.cc | 20 +- .../upload/upload_client_unittest.cc | 1 + chrome/test/BUILD.gn | 2 +- .../summary/annotations.xml | 1 + tools/traffic_annotation/summary/grouping.xml | 1 + 10 files changed, 1659 insertions(+), 34 deletions(-) diff --git a/chrome/browser/BUILD.gn b/chrome/browser/BUILD.gn index 5d875cbc06367..23fb26e3d3a49 100644 --- a/chrome/browser/BUILD.gn +++ b/chrome/browser/BUILD.gn @@ -1189,8 +1189,6 @@ static_library("browser") { "policy/messaging_layer/upload/encrypted_reporting_client.h", "policy/messaging_layer/upload/event_upload_size_controller.cc", "policy/messaging_layer/upload/event_upload_size_controller.h", - "policy/messaging_layer/upload/file_upload_impl.cc", - "policy/messaging_layer/upload/file_upload_impl.h", "policy/messaging_layer/upload/file_upload_job.cc", "policy/messaging_layer/upload/file_upload_job.h", "policy/messaging_layer/upload/network_condition_service.cc", @@ -4887,6 +4885,8 @@ static_library("browser") { "platform_util_ash.cc", "policy/default_geolocation_policy_handler.cc", "policy/default_geolocation_policy_handler.h", + "policy/messaging_layer/upload/file_upload_impl.cc", + "policy/messaging_layer/upload/file_upload_impl.h", "policy/os_color_mode_policy_handler.cc", "policy/os_color_mode_policy_handler.h", "policy/status_provider/device_active_directory_policy_status_provider.cc", diff --git a/chrome/browser/policy/messaging_layer/upload/file_upload_impl.cc b/chrome/browser/policy/messaging_layer/upload/file_upload_impl.cc index c63e8b20784d4..f4272f45bfb32 100644 --- a/chrome/browser/policy/messaging_layer/upload/file_upload_impl.cc +++ b/chrome/browser/policy/messaging_layer/upload/file_upload_impl.cc @@ -6,20 +6,934 @@ #include "chrome/browser/policy/messaging_layer/upload/file_upload_impl.h" #include +#include +#include "base/files/file.h" +#include "base/files/file_util.h" +#include "base/memory/ptr_util.h" +#include "base/sequence_checker.h" +#include "base/strings/strcat.h" #include "base/strings/string_piece.h" +#include "base/strings/string_split.h" +#include "base/strings/string_util.h" +#include "base/task/bind_post_task.h" +#include "base/task/task_traits.h" +#include "base/task/thread_pool.h" +#include "base/thread_annotations.h" +#include "chrome/browser/browser_process.h" +#include "chrome/browser/device_identity/device_oauth2_token_service.h" +#include "chrome/browser/device_identity/device_oauth2_token_service_factory.h" +#include "chrome/browser/policy/chrome_browser_policy_connector.h" +#include "chrome/browser/policy/messaging_layer/upload/file_upload_job.h" #include "components/reporting/util/status.h" +#include "content/public/browser/browser_task_traits.h" +#include "content/public/browser/browser_thread.h" +#include "google_apis/gaia/core_account_id.h" +#include "google_apis/gaia/gaia_constants.h" +#include "google_apis/gaia/gaia_oauth_client.h" +#include "google_apis/gaia/google_service_auth_error.h" +#include "google_apis/gaia/oauth2_access_token_manager.h" +#include "net/base/net_errors.h" +#include "net/http/http_request_headers.h" +#include "net/http/http_response_headers.h" +#include "net/http/http_status_code.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "services/network/public/cpp/resource_request.h" +#include "services/network/public/cpp/shared_url_loader_factory.h" +#include "services/network/public/cpp/simple_url_loader.h" +#include "url/gurl.h" namespace reporting { + +constexpr char kAuthorizationPrefix[] = "Bearer "; + +constexpr char kUploadStatusHeader[] = "X-Goog-Upload-Status"; +constexpr char kUploadCommandHeader[] = "X-Goog-Upload-Command"; +constexpr char kUploadHeaderContentLengthHeader[] = + "X-Goog-Upload-Header-Content-Length"; +constexpr char kUploadHeaderContentTypeHeader[] = + "X-Goog-Upload-Header-Content-Type"; +constexpr char kUploadChunkGranularityHeader[] = + "X-Goog-Upload-Chunk-Granularity"; +constexpr char kUploadUrlHeader[] = "X-Goog-Upload-Url"; +constexpr char kUploadSizeReceivedHeader[] = "X-Goog-Upload-Size-Received"; +constexpr char kUploadOffsetHeader[] = "X-Goog-Upload-Offset"; +constexpr char kUploadProtocolHeader[] = "X-Goog-Upload-Protocol"; +constexpr char kUploadIdHeader[] = "X-GUploader-UploadID"; + +// Deletes original file (called on a thread pool upon successful upload). +void DeleteOriginalFile(const std::string origin_path) { + const auto delete_result = base::DeleteFile(base::FilePath(origin_path)); + if (!delete_result) { + LOG(WARNING) << "Failed to delete file=" << origin_path; + } +} + +// Helper for network response, headers analysis and status retrieval. +StatusOr CheckResponseAndGetStatus( + const std::unique_ptr<::network::SimpleURLLoader> url_loader, + const scoped_refptr<::net::HttpResponseHeaders> headers) { + if (!headers) { + return Status(error::DATA_LOSS, + base::StrCat({"Network error=", + ::net::ErrorToString(url_loader->NetError())})); + } + + if (headers->response_code() == net::HTTP_OK) { + // Successful upload, retrieve and return upload status. + std::string upload_status; + if (!headers->GetNormalizedHeader(kUploadStatusHeader, &upload_status)) { + return Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload status=", upload_status})); + } + return upload_status; + } else if (headers->response_code() == net::HTTP_UNAUTHORIZED) { + return Status(error::UNAUTHENTICATED, "Authentication error"); + } else { + return Status( + error::DATA_LOSS, + base::StrCat({"POST request failed with HTTP status code ", + base::NumberToString(headers->response_code())})); + } +} + +// Helper to learn upload chunk granularity. +StatusOr GetChunkGranularity( + const scoped_refptr<::net::HttpResponseHeaders> headers) { + int64_t upload_granularity = -1; + std::string upload_granularity_string; + if (!headers->GetNormalizedHeader(kUploadChunkGranularityHeader, + &upload_granularity_string) || + !base::StringToInt64(upload_granularity_string, &upload_granularity) || + upload_granularity <= 0L) { + return Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload granularity=", + upload_granularity_string})); + } + return upload_granularity; +} + +// Generic context that returns result and self-destructs. +template +class ActionContext { + public: + ActionContext(const ActionContext& other) = delete; + ActionContext& operator=(const ActionContext& other) = delete; + virtual ~ActionContext() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(!result_cb_) << "Destruct before callback"; + } + + protected: + // Constructor is only available to derived classes. + ActionContext(base::WeakPtr delegate, + base::OnceCallback result_cb) + : delegate_(std::move(delegate)), result_cb_(std::move(result_cb)) {} + + // Completes work returning result or status, and then self-destructs. + void Complete(R result) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(result_cb_) << "Already completed"; + std::move(result_cb_).Run(std::move(result)); + delete this; + } + + // Accessor. + base::WeakPtr delegate() const { return delegate_; } + + SEQUENCE_CHECKER(sequence_checker_); + + private: + const base::WeakPtr delegate_; + base::OnceCallback result_cb_ GUARDED_BY_CONTEXT(sequence_checker_); +}; + +// Self-destructing context for Authentication. +class FileUploadDelegate::AccessTokenRetriever + : public ActionContext>, + public OAuth2AccessTokenManager::Consumer { + public: + AccessTokenRetriever( + base::WeakPtr delegate, + base::OnceCallback)> result_cb) + : ActionContext(std::move(delegate), std::move(result_cb)), + OAuth2AccessTokenManager::Consumer("cros_upload_job") {} + + void RequestAccessToken() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + DCHECK(!access_token_request_); + DVLOG(1) << "Requesting access token."; + + access_token_request_ = delegate()->StartOAuth2Request(this); + } + + private: + // OAuth2AccessTokenManager::Consumer: + void OnGetTokenSuccess( + const OAuth2AccessTokenManager::Request* request, + const OAuth2AccessTokenConsumer::TokenResponse& token_response) override { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK_EQ(access_token_request_.get(), request); + access_token_request_.reset(); + DVLOG(1) << "Token successfully acquired."; + Complete(token_response.access_token); + } + + void OnGetTokenFailure(const OAuth2AccessTokenManager::Request* request, + const GoogleServiceAuthError& error) override { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK_EQ(access_token_request_.get(), request); + access_token_request_.reset(); + LOG(ERROR) << "Token request failed: " << error.ToString(); + Complete(Status(error::UNAUTHENTICATED, error.ToString())); + } + + // The OAuth request to receive the access token. + std::unique_ptr access_token_request_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // Should remain the last member so it will be destroyed first and + // invalidate all weak pointers. + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + +// Self-destructing context for FileUploadJob initiation. +class FileUploadDelegate::InitContext + : public ActionContext>> { + public: + InitContext( + base::StringPiece origin_path, + base::StringPiece upload_parameters, + base::StringPiece access_token, + base::WeakPtr delegate, + base::OnceCallback< + void(StatusOr>)> result_cb) + : ActionContext(std::move(delegate), std::move(result_cb)), + origin_path_(origin_path), + upload_parameters_(upload_parameters), + access_token_(access_token) {} + + void Run() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + // Perform file operation on a thread pool, then resume on the current task + // runner. + base::ThreadPool::PostTaskAndReplyWithResult( + FROM_HERE, {base::TaskPriority::BEST_EFFORT, base::MayBlock()}, + base::BindOnce(&InitContext::InitFile, origin_path_), + base::BindOnce(&InitContext::FileOpened, + weak_ptr_factory_.GetWeakPtr())); + } + + void FileOpened(StatusOr total_result) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + if (!total_result.ok()) { + Complete(total_result.status()); + return; + } + + // Record total size of the file. + total_ = total_result.ValueOrDie(); + + // Extract base name of the file for metadata. + const base::FilePath full_name = base::FilePath(origin_path_); + const auto base_name = full_name.BaseName().MaybeAsASCII(); + if (base_name.empty()) { + Complete(Status(error::FAILED_PRECONDITION, + base::StrCat({"Origin path malformed: ", origin_path_}))); + return; + } + + // Initiate upload. + DVLOG(1) << "Starting URL fetcher."; + + auto resource_request = std::make_unique<::network::ResourceRequest>(); + resource_request->headers.SetHeader( + ::net::HttpRequestHeaders::kAuthorization, + base::StrCat({kAuthorizationPrefix, access_token_.c_str()})); + resource_request->headers.SetHeader(kUploadCommandHeader, "start"); + resource_request->headers.SetHeader(kUploadHeaderContentLengthHeader, + base::NumberToString(total_)); + resource_request->headers.SetHeader(kUploadHeaderContentTypeHeader, + "application/octet-stream"); + + url_loader_ = delegate()->CreatePostLoader(std::move(resource_request)); + + // Construct and attach medatata. + // See go/scotty-http-protocols#unified-resumable-protocol + // Retrieve the base name of the file to be used as uploaded file name. + // TODO(b/264399295): The rest is to be populated from the event. + std::string metadata; + metadata.append("\r\n") + .append(" ") + .append("support_file") // TODO(b/264399295): get from the event + .append("\r\n") + .append("\r\n"); + metadata.append("\r\n") + .append(" ") + .append("ID12345") // TODO(b/264399295): get from the event + .append("\r\n") + .append("\r\n"); + metadata.append("\r\n") + .append(" ") + .append(base_name) + .append("\r\n") + .append("\r\n"); + url_loader_->AttachStringForUpload(metadata, "text/xml"); + + // Make a call and get response headers. + delegate()->SendAndGetResponse( + url_loader_.get(), base::BindOnce(&InitContext::OnInitURLLoadComplete, + weak_ptr_factory_.GetWeakPtr())); + } + + void OnInitURLLoadComplete( + scoped_refptr<::net::HttpResponseHeaders> headers) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + auto status_result = + CheckResponseAndGetStatus(std::move(url_loader_), headers); + if (!status_result.ok()) { + Complete(status_result.status()); + return; + } + + const std::string upload_status = status_result.ValueOrDie(); + if (!base::EqualsCaseInsensitiveASCII(upload_status, "active")) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload status=", upload_status}))); + return; + } + + // Just make sure granulatiy is returned, do not use it here. + auto upload_granularity_result = GetChunkGranularity(headers); + if (!upload_granularity_result.ok()) { + Complete(upload_granularity_result.status()); + return; + } + + std::string upload_url; + if (!headers->GetNormalizedHeader(kUploadUrlHeader, &upload_url)) { + Complete(Status(error::DATA_LOSS, "No upload URL returned")); + return; + } + + Complete( + std::make_pair(total_, base::StrCat({origin_path_, "\n", upload_url}))); + } + + static StatusOr InitFile(const std::string origin_path) { + auto handle = std::make_unique( + base::FilePath(origin_path), + base::File::FLAG_OPEN | base::File::FLAG_READ); + if (!handle->IsValid()) { + return Status( + error::DATA_LOSS, + base::StrCat({"Cannot open file=", origin_path, " error=", + base::File::ErrorToString(handle->error_details())})); + } + + // Calculate total size of the file. + return handle->GetLength(); + } + + private: + const std::string origin_path_; + const std::string upload_parameters_; + const std::string access_token_; + + // Helper to upload the data. + std::unique_ptr<::network::SimpleURLLoader> url_loader_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // Total size. + int64_t total_ GUARDED_BY_CONTEXT(sequence_checker_) = 0L; + + // Should remain the last member so it will be destroyed first and + // invalidate all weak pointers. + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + +// Self-destructing context for FileUploadJob next step. +class FileUploadDelegate::NextStepContext + : public ActionContext>> { + public: + NextStepContext( + int64_t total, + int64_t uploaded, + base::StringPiece session_token, + base::WeakPtr delegate, + base::OnceCallback< + void(StatusOr>)> result_cb) + : ActionContext(std::move(delegate), std::move(result_cb)), + total_(total), + uploaded_(uploaded), + session_token_(session_token) {} + + void Run() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + // Parse session token. + const auto tokens = base::SplitStringPiece( + session_token_, "\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_ALL); + if (tokens.size() != 2 || tokens[0].empty() || tokens[1].empty()) { + Complete(Status(error::DATA_LOSS, base::StrCat({"Corrupt session token `", + session_token_, "`"}))); + return; + } + origin_path_ = tokens[0]; + resumable_upload_url_ = GURL(tokens[1]); + if (!resumable_upload_url_.is_valid()) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Corrupt resumable upload URL=", tokens[1]}))); + return; + } + + // Query upload. + DVLOG(1) << "Starting Query URL fetcher."; + auto resource_request = std::make_unique<::network::ResourceRequest>(); + resource_request->url = resumable_upload_url_; + resource_request->headers.SetHeader(kUploadCommandHeader, "query"); + + url_loader_ = delegate()->CreatePostLoader(std::move(resource_request)); + + // Make a call and get response headers. + delegate()->SendAndGetResponse( + url_loader_.get(), + base::BindOnce(&NextStepContext::OnQueryURLLoadComplete, + weak_ptr_factory_.GetWeakPtr())); + } + + void OnQueryURLLoadComplete( + scoped_refptr<::net::HttpResponseHeaders> headers) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + auto status_result = + CheckResponseAndGetStatus(std::move(url_loader_), headers); + if (!status_result.ok()) { + Complete(status_result.status()); + return; + } + + const std::string upload_status = status_result.ValueOrDie(); + if (base::EqualsCaseInsensitiveASCII(upload_status, "final")) { + // Already done. + Complete(std::make_pair(total_, session_token_)); + return; + } + if (!base::EqualsCaseInsensitiveASCII(upload_status, "active")) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload status=", upload_status}))); + return; + } + + int64_t upload_received = -1; + { + std::string upload_received_string; + if (!headers->GetNormalizedHeader(kUploadSizeReceivedHeader, + &upload_received_string)) { + Complete(Status(error::DATA_LOSS, "No upload size returned")); + return; + } + if (!base::StringToInt64(upload_received_string, &upload_received) || + upload_received < 0 || uploaded_ > upload_received) { + Complete(Status( + error::DATA_LOSS, + base::StrCat( + {"Unexpected received=", base::NumberToString(upload_received), + ", expected=", base::NumberToString(uploaded_)}))); + return; + } + } + if (upload_received >= total_) { + // Already done. + Complete(std::make_pair(total_, session_token_)); + return; + } + + auto upload_granularity_result = GetChunkGranularity(headers); + if (!upload_granularity_result.ok()) { + Complete(upload_granularity_result.status()); + return; + } + auto upload_granularity = upload_granularity_result.ValueOrDie(); + + // Determine maximum buffer size, rounded down to upload_granularity. + DCHECK_CALLED_ON_VALID_SEQUENCE(delegate()->sequence_checker_); + int64_t max_size = + (delegate()->max_upload_buffer_size_ / upload_granularity) * + upload_granularity; + + // Upload next or last chunk. + DVLOG(1) << "Starting Upload URL fetcher."; + auto resource_request = std::make_unique<::network::ResourceRequest>(); + resource_request->url = resumable_upload_url_; + int64_t size = total_ - upload_received; + if (size < max_size) { + resource_request->headers.SetHeader(kUploadCommandHeader, + "upload, finalize"); + } else { + size = max_size; + resource_request->headers.SetHeader(kUploadCommandHeader, "upload"); + } + resource_request->headers.SetHeader(kUploadOffsetHeader, + base::NumberToString(upload_received)); + + // Retrieve data from the file to be attached on a thread pool, then resume + // on the current task runner. Note: it could be done with + // `AttachFileForUpload` instead, but loading into memory allows to check + // integrity of the file (TBD; for now we only verify file access and + // size). + base::ThreadPool::PostTaskAndReplyWithResult( + FROM_HERE, {base::TaskPriority::BEST_EFFORT, base::MayBlock()}, + base::BindOnce(&NextStepContext::LoadFileData, + std::string(origin_path_), total_, upload_received, + size), + base::BindOnce(&NextStepContext::PerformUpload, + weak_ptr_factory_.GetWeakPtr(), upload_received, size, + std::move(resource_request))); + } + + void PerformUpload( + int64_t upload_received, + int64_t size, + std::unique_ptr<::network::ResourceRequest> resource_request, + StatusOr buffer_result) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + if (!buffer_result.ok()) { + Complete(buffer_result.status()); + return; + } + + url_loader_ = delegate()->CreatePostLoader(std::move(resource_request)); + url_loader_->AttachStringForUpload( + buffer_result.ValueOrDie(), // owned by caller! + "application/octet-stream"); + + // Make a call and get response headers. + delegate()->SendAndGetResponse( + url_loader_.get(), + base::BindOnce(&NextStepContext::OnUploadURLLoadComplete, + weak_ptr_factory_.GetWeakPtr(), upload_received, size)); + } + + void OnUploadURLLoadComplete( + int64_t uploaded, + int64_t size, + scoped_refptr<::net::HttpResponseHeaders> headers) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + auto status_result = + CheckResponseAndGetStatus(std::move(url_loader_), headers); + if (!status_result.ok()) { + Complete(status_result.status()); + return; + } + + const std::string upload_status = status_result.ValueOrDie(); + if (base::EqualsCaseInsensitiveASCII(upload_status, "final")) { + // Already done. + Complete(std::make_pair(total_, session_token_)); + return; + } + if (!base::EqualsCaseInsensitiveASCII(upload_status, "active")) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload status=", upload_status}))); + return; + } + + Complete(std::make_pair(uploaded + size, session_token_)); + } + + static StatusOr LoadFileData(const std::string origin_path, + int64_t total, + int64_t offset, + int64_t size) { + // Retrieve data from the file to be attached. Note: it could be done with + // `AttachFileForUpload` instead, but loading into memory allows to check + // integrity of the file (TBD; for now we only verify file access and + // size). + std::string buffer; + + auto handle = std::make_unique( + base::FilePath(origin_path), + base::File::FLAG_OPEN | base::File::FLAG_READ); + if (!handle->IsValid()) { + return Status( + error::DATA_LOSS, + base::StrCat({"Cannot open file=", origin_path, " error=", + base::File::ErrorToString(handle->error_details())})); + } + + // Verify total size of the file. + if (total != handle->GetLength()) { + return Status(error::DATA_LOSS, + base::StrCat({"File=", origin_path, " changed size ", + " from ", base::NumberToString(total), " to ", + base::NumberToString(handle->GetLength())})); + } + + // Load into buffer. TODO(b/264399295): Add memory resource control. + buffer.resize( + size); // Initialization is redundant, but std::string mandates it. + const int read_size = handle->Read(offset, buffer.data(), size); + if (read_size < 0) { + return Status( + error::DATA_LOSS, + base::StrCat({"Cannot read file=", origin_path, " error=", + base::File::ErrorToString(handle->error_details())})); + } + if (read_size != size) { + return Status(error::DATA_LOSS, + base::StrCat({"Failed to read file=", origin_path, + " offset=", base::NumberToString(offset), + " size=", base::NumberToString(size), + " read=", base::NumberToString(read_size)})); + } + return buffer; + } + + private: + const int64_t total_; + const int64_t uploaded_; + const std::string session_token_; + + // Session token components. + base::StringPiece origin_path_ GUARDED_BY_CONTEXT(sequence_checker_); + GURL resumable_upload_url_ GUARDED_BY_CONTEXT(sequence_checker_); + + // Helper to upload the data. + std::unique_ptr url_loader_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // Should remain the last member so it will be destroyed first and + // invalidate all weak pointers. + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + +// Self-destructing context for FileUploadJob finalization. +class FileUploadDelegate::FinalContext + : public ActionContext> { + public: + FinalContext( + base::StringPiece session_token, + base::WeakPtr delegate, + base::OnceCallback)> + result_cb) + : ActionContext(std::move(delegate), std::move(result_cb)), + session_token_(session_token) {} + + void Run() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + // Parse session token. + const auto tokens = base::SplitStringPiece( + session_token_, "\n", base::KEEP_WHITESPACE, base::SPLIT_WANT_ALL); + if (tokens.size() != 2 || tokens[0].empty() || tokens[1].empty()) { + Complete(Status(error::DATA_LOSS, base::StrCat({"Corrupt session token `", + session_token_, "`"}))); + return; + } + origin_path_ = tokens[0]; + resumable_upload_url_ = GURL(tokens[1]); + if (!resumable_upload_url_.is_valid()) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Corrupt resumable upload URL=", tokens[1]}))); + return; + } + + // Query upload. + DVLOG(1) << "Starting Query URL fetcher."; + auto resource_request = std::make_unique<::network::ResourceRequest>(); + resource_request->url = resumable_upload_url_; + resource_request->headers.SetHeader(kUploadCommandHeader, "query"); + + url_loader_ = delegate()->CreatePostLoader(std::move(resource_request)); + + // Make a call and get response headers. + delegate()->SendAndGetResponse( + url_loader_.get(), base::BindOnce(&FinalContext::OnQueryURLLoadComplete, + weak_ptr_factory_.GetWeakPtr())); + } + + void OnQueryURLLoadComplete( + scoped_refptr<::net::HttpResponseHeaders> headers) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK(delegate()); + + auto status_result = + CheckResponseAndGetStatus(std::move(url_loader_), headers); + if (!status_result.ok()) { + Complete(status_result.status()); + return; + } + + const std::string upload_status = status_result.ValueOrDie(); + if (base::EqualsCaseInsensitiveASCII(upload_status, "final")) { + // All done. + RespondOnFinal(headers); + return; + } + if (!base::EqualsCaseInsensitiveASCII(upload_status, "active")) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload status=", upload_status}))); + return; + } + + int64_t upload_received = -1; + { + std::string upload_received_string; + if (!headers->GetNormalizedHeader(kUploadSizeReceivedHeader, + &upload_received_string)) { + Complete(Status(error::DATA_LOSS, "No upload size returned")); + return; + } + if (!base::StringToInt64(upload_received_string, &upload_received) || + upload_received < 0) { + Complete(Status(error::DATA_LOSS, + base::StrCat({"Unexpected received=", + base::NumberToString(upload_received)}))); + return; + } + } + + // Finalize upload. + DVLOG(1) << "Starting Upload URL fetcher."; + auto resource_request = std::make_unique<::network::ResourceRequest>(); + resource_request->url = resumable_upload_url_; + resource_request->headers.SetHeader(kUploadCommandHeader, "finalize"); + + url_loader_ = delegate()->CreatePostLoader(std::move(resource_request)); + + // Make a call and get response headers. + delegate()->SendAndGetResponse( + url_loader_.get(), + base::BindOnce(&FinalContext::OnFinalizeURLLoadComplete, + weak_ptr_factory_.GetWeakPtr())); + } + + void OnFinalizeURLLoadComplete( + scoped_refptr<::net::HttpResponseHeaders> headers) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + auto status_result = + CheckResponseAndGetStatus(std::move(url_loader_), headers); + if (!status_result.ok()) { + Complete(status_result.status()); + return; + } + + const std::string upload_status = status_result.ValueOrDie(); + if (!base::EqualsCaseInsensitiveASCII(upload_status, "final")) { + Complete( + Status(error::DATA_LOSS, + base::StrCat({"Unexpected upload status=", upload_status}))); + return; + } + + RespondOnFinal(headers); + } + + private: + void RespondOnFinal(scoped_refptr<::net::HttpResponseHeaders> headers) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + + std::string upload_id; + if (!headers->GetNormalizedHeader(kUploadIdHeader, &upload_id) || + upload_id.empty()) { + Complete(Status(error::DATA_LOSS, "No upload ID returned")); + return; + } + + // Delete file upon success (on a thread pool, don't wait for completion). + base::ThreadPool::PostTask( + FROM_HERE, {base::TaskPriority::BEST_EFFORT, base::MayBlock()}, + base::BindOnce(&DeleteOriginalFile, std::string(origin_path_))); + + Complete(base::StrCat({"Upload_id=", upload_id})); + } + + const std::string session_token_; + + // Session token components. + base::StringPiece origin_path_ GUARDED_BY_CONTEXT(sequence_checker_); + GURL resumable_upload_url_ GUARDED_BY_CONTEXT(sequence_checker_); + + // Helper to upload the data. + std::unique_ptr url_loader_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // Should remain the last member so it will be destroyed first and + // invalidate all weak pointers. + base::WeakPtrFactory weak_ptr_factory_{this}; +}; + FileUploadDelegate::FileUploadDelegate() = default; +FileUploadDelegate::~FileUploadDelegate() = default; + +void FileUploadDelegate::InitializeOnce() { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + DCHECK_CURRENTLY_ON(::content::BrowserThread::UI); + if (url_loader_factory_) { + return; // Already initialized. + } + + static constexpr char kLogUploadUrlTail[] = "/upload"; + upload_url_ = GURL( + g_browser_process->browser_policy_connector()->GetDeviceManagementUrl() + + kLogUploadUrlTail); + DCHECK(upload_url_.is_valid()); + + account_id_ = DeviceOAuth2TokenServiceFactory::Get()->GetRobotAccountId(); + access_token_manager_ = + DeviceOAuth2TokenServiceFactory::Get()->GetAccessTokenManager(); + DCHECK(access_token_manager_); + url_loader_factory_ = g_browser_process->shared_url_loader_factory(); + DCHECK(url_loader_factory_); + traffic_annotation_ = std::make_unique<::net::NetworkTrafficAnnotationTag>( + ::net::DefineNetworkTrafficAnnotation("chrome_support_tool_file_upload", + R"( + semantics { + sender: "ChromeOS Support Tool" + description: + "ChromeOS Support Tool can request log files upload on a managed " + "device on behalf of the admin. The log files are bundled " + "together in a single zip archive that is uploaded using " + "a multi-chunk resumable protocol. They are stored on Google " + "servers, so the admin can view the logs in the Admin Console." + trigger: "When UPLOAD_LOG event is posted by Support Tool " + "on the admin's request." + data: "Zipped archive of log files created by Support Tool and " + "placed in /var/spool/." + destination: GOOGLE_OWNED_SERVICE + internal { + contacts { + email: "lbaraz@chromium.org" + } + contacts { + email: "cros-reporting-team@google.com" + } + } + user_data { + type: USER_CONTENT + } + last_reviewed: "2023-03-16" + } + policy { + cookies_allowed: NO + setting: "This feature cannot be disabled in settings." + chrome_device_policy { + # LogUploadEnabled + device_log_upload_settings { + system_log_upload_enabled: false + } + } + } + )")); + + max_upload_buffer_size_ = 1L * 1024L * 1024L; // 1 MiB +} + +std::unique_ptr +FileUploadDelegate::StartOAuth2Request( + OAuth2AccessTokenManager::Consumer* consumer) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + OAuth2AccessTokenManager::ScopeSet scope_set; + scope_set.insert(GaiaConstants::kDeviceManagementServiceOAuth); + return access_token_manager_->StartRequest(account_id_, scope_set, consumer); +} + +std::unique_ptr<::network::SimpleURLLoader> +FileUploadDelegate::CreatePostLoader( + std::unique_ptr<::network::ResourceRequest> resource_request) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + resource_request->method = "POST"; + if (resource_request->url.is_empty()) { + resource_request->url = upload_url_; + } + resource_request->headers.SetHeader(kUploadProtocolHeader, "resumable"); + return ::network::SimpleURLLoader::Create(std::move(resource_request), + *traffic_annotation_); +} + +void FileUploadDelegate::SendAndGetResponse( + ::network::SimpleURLLoader* url_loader, + base::OnceCallback headers)> + response_cb) const { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + url_loader->DownloadHeadersOnly( + url_loader_factory_.get(), + base::BindPostTaskToCurrentDefault(std::move(response_cb))); +} + +// static void FileUploadDelegate::DoInitiate( base::StringPiece origin_path, base::StringPiece upload_parameters, base::OnceCallback>)> - cb) { - std::move(cb).Run(Status(error::UNIMPLEMENTED, "Not yet implemented")); + result_cb) { + if (!::content::BrowserThread::CurrentlyOn(::content::BrowserThread::UI)) { + ::content::GetUIThreadTaskRunner({})->PostTask( + FROM_HERE, + base::BindOnce(&FileUploadDelegate::DoInitiate, GetWeakPtr(), + std::string(origin_path), std::string(upload_parameters), + std::move(result_cb))); + return; + } + + InitializeOnce(); + + DVLOG(1) << "Creating file upload job for support tool use"; + + (new AccessTokenRetriever( + GetWeakPtr(), + base::BindPostTaskToCurrentDefault(base::BindOnce( + &FileUploadDelegate::OnAccessTokenResult, GetWeakPtr(), + std::string(origin_path), std::string(upload_parameters), + std::move(result_cb))))) + ->RequestAccessToken(); +} + +void FileUploadDelegate::OnAccessTokenResult( + base::StringPiece origin_path, + base::StringPiece upload_parameters, + base::OnceCallback>)> + result_cb, + StatusOr access_token_result) { + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + if (!access_token_result.ok()) { + std::move(result_cb).Run(access_token_result.status()); + return; + } + + // Measure file size and store it in total. + (new InitContext(origin_path, upload_parameters, + access_token_result.ValueOrDie(), GetWeakPtr(), + std::move(result_cb))) + ->Run(); } void FileUploadDelegate::DoNextStep( @@ -28,13 +942,40 @@ void FileUploadDelegate::DoNextStep( base::StringPiece session_token, base::OnceCallback>)> - cb) { - std::move(cb).Run(Status(error::UNIMPLEMENTED, "Not yet implemented")); + result_cb) { + if (!::content::BrowserThread::CurrentlyOn(::content::BrowserThread::UI)) { + ::content::GetUIThreadTaskRunner({})->PostTask( + FROM_HERE, base::BindOnce(&FileUploadDelegate::DoNextStep, GetWeakPtr(), + total, uploaded, std::string(session_token), + std::move(result_cb))); + return; + } + + InitializeOnce(); + + (new NextStepContext(total, uploaded, session_token, GetWeakPtr(), + std::move(result_cb))) + ->Run(); } void FileUploadDelegate::DoFinalize( base::StringPiece session_token, - base::OnceCallback)> cb) { - std::move(cb).Run(Status(error::UNIMPLEMENTED, "Not yet implemented")); + base::OnceCallback)> + result_cb) { + if (!::content::BrowserThread::CurrentlyOn(::content::BrowserThread::UI)) { + ::content::GetUIThreadTaskRunner({})->PostTask( + FROM_HERE, + base::BindOnce(&FileUploadDelegate::DoFinalize, GetWeakPtr(), + std::string(session_token), std::move(result_cb))); + return; + } + + InitializeOnce(); + + (new FinalContext(session_token, GetWeakPtr(), std::move(result_cb)))->Run(); +} + +base::WeakPtr FileUploadDelegate::GetWeakPtr() { + return weak_ptr_factory_.GetWeakPtr(); } } // namespace reporting diff --git a/chrome/browser/policy/messaging_layer/upload/file_upload_impl.h b/chrome/browser/policy/messaging_layer/upload/file_upload_impl.h index 93e08efb4df1b..39a8caed82e08 100644 --- a/chrome/browser/policy/messaging_layer/upload/file_upload_impl.h +++ b/chrome/browser/policy/messaging_layer/upload/file_upload_impl.h @@ -6,32 +6,50 @@ #define CHROME_BROWSER_POLICY_MESSAGING_LAYER_UPLOAD_FILE_UPLOAD_IMPL_H_ #include +#include +#include "base/files/file.h" +#include "base/memory/raw_ptr.h" +#include "base/memory/weak_ptr.h" +#include "base/sequence_checker.h" #include "base/strings/string_piece.h" +#include "base/task/sequenced_task_runner.h" +#include "base/thread_annotations.h" +#include "chrome/browser/browser_process.h" #include "chrome/browser/policy/messaging_layer/upload/file_upload_job.h" #include "components/reporting/util/status.h" +#include "google_apis/gaia/core_account_id.h" +#include "google_apis/gaia/oauth2_access_token_manager.h" +#include "net/http/http_response_headers.h" +#include "net/traffic_annotation/network_traffic_annotation.h" +#include "services/network/public/cpp/simple_url_loader.h" +#include "url/gurl.h" namespace reporting { class FileUploadDelegate : public FileUploadJob::Delegate { public: FileUploadDelegate(); + ~FileUploadDelegate() override; + + static std::string GetFileUploadUrl(); private: - // Delegate implementation. - // Asynchronously initializes upload. - // Calls back with `total` and `session_token` are set, or Status in case - // of error. + // Helper classes. + class AccessTokenRetriever; + class InitContext; + class NextStepContext; + class FinalContext; + + friend class FileUploadDelegateTest; + + // FileUploadJob::Delegate: void DoInitiate( base::StringPiece origin_path, base::StringPiece upload_parameters, base::OnceCallback>)> cb) override; - - // Asynchronously uploads the next chunk. - // Calls back with new `uploaded` and `session_token` (could be the same), - // or Status in case of an error. void DoNextStep( int64_t total, int64_t uploaded, @@ -39,13 +57,69 @@ class FileUploadDelegate : public FileUploadJob::Delegate { base::OnceCallback>)> cb) override; - - // Asynchronously finalizes upload (once `uploaded` reached `total`). - // Calls back with `access_parameters`, or Status in case of error. void DoFinalize( - base::StringPiece session_token, + base::StringPiece access_parameters, base::OnceCallback)> cb) override; + + // Called once authentication is finished (with token or failure status). + void OnAccessTokenResult( + base::StringPiece origin_path, + base::StringPiece upload_parameters, + base::OnceCallback>)> cb, + StatusOr access_token_result); + + // Initializes the delegate once, lazily - when the first API is called. + // On later API calls this method immediately returns. + void InitializeOnce(); + + // Helper method starts OAuth2 token request. + [[nodiscard]] std::unique_ptr + StartOAuth2Request( + OAuth2AccessTokenManager::Consumer* consumer // owned by the caller! + ) const; + + // Helper method populates rest of request and creates SimpleURLLoader + // instance. + [[nodiscard]] std::unique_ptr<::network::SimpleURLLoader> CreatePostLoader( + std::unique_ptr<::network::ResourceRequest> resource_request) const; + + // Helper method sends request and hands the response headers to `response_cb` + // (on the current task runner). + void SendAndGetResponse( + ::network::SimpleURLLoader* url_loader, // owned by the caller! + base::OnceCallback + headers)> response_cb) const; + + base::WeakPtr GetWeakPtr(); + + SEQUENCE_CHECKER(sequence_checker_); + + // The URL to which the POST request should be directed. + GURL upload_url_ GUARDED_BY_CONTEXT(sequence_checker_); + + // The account ID that will be used for the access token fetch. + CoreAccountId account_id_ GUARDED_BY_CONTEXT(sequence_checker_); + + // The token manager used to retrieve the access token (not owned). + raw_ptr access_token_manager_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // This is used to initialize the network::SimpleURLLoader object. + scoped_refptr<::network::SharedURLLoaderFactory> url_loader_factory_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // Network traffic annotation set by the delegate describing what kind of data + // is uploaded. + std::unique_ptr traffic_annotation_ + GUARDED_BY_CONTEXT(sequence_checker_); + + // Maximum upload size allowed for a single request. + int64_t max_upload_buffer_size_ GUARDED_BY_CONTEXT(sequence_checker_); + + base::WeakPtrFactory weak_ptr_factory_{this}; }; } // namespace reporting diff --git a/chrome/browser/policy/messaging_layer/upload/file_upload_impl_unittest.cc b/chrome/browser/policy/messaging_layer/upload/file_upload_impl_unittest.cc index 12afc68eea475..c8c143d60140a 100644 --- a/chrome/browser/policy/messaging_layer/upload/file_upload_impl_unittest.cc +++ b/chrome/browser/policy/messaging_layer/upload/file_upload_impl_unittest.cc @@ -4,27 +4,620 @@ #include "chrome/browser/policy/messaging_layer/upload/file_upload_impl.h" +#include + +#include +#include #include +#include -#include "base/test/task_environment.h" -#include "chrome/browser/policy/messaging_layer/upload/file_upload_job_test_util.h" +#include "base/containers/contains.h" +#include "base/containers/queue.h" +#include "base/files/file.h" +#include "base/files/file_util.h" +#include "base/files/scoped_temp_dir.h" +#include "base/functional/bind.h" +#include "base/functional/callback_helpers.h" +#include "base/location.h" +#include "base/memory/ptr_util.h" +#include "base/sequence_checker.h" +#include "base/strings/strcat.h" +#include "base/task/single_thread_task_runner.h" +#include "base/time/time.h" +#include "chrome/browser/ash/policy/uploading/upload_job_impl.h" #include "components/reporting/proto/synced/upload_tracker.pb.h" +#include "components/reporting/util/status.h" +#include "components/reporting/util/statusor.h" +#include "components/reporting/util/test_support_callbacks.h" +#include "content/public/test/browser_task_environment.h" +#include "google_apis/gaia/core_account_id.h" +#include "google_apis/gaia/fake_oauth2_access_token_manager.h" +#include "google_apis/gaia/gaia_access_token_fetcher.h" +#include "google_apis/gaia/google_service_auth_error.h" +#include "google_apis/gaia/oauth2_access_token_manager.h" +#include "net/http/http_status_code.h" +#include "net/test/embedded_test_server/embedded_test_server.h" +#include "net/test/embedded_test_server/http_request.h" +#include "net/test/embedded_test_server/http_response.h" +#include "net/traffic_annotation/network_traffic_annotation_test_helper.h" +#include "services/network/test/test_network_context.h" +#include "services/network/test/test_network_context_client.h" +#include "services/network/test/test_shared_url_loader_factory.h" +#include "testing/gmock/include/gmock/gmock-matchers.h" #include "testing/gmock/include/gmock/gmock.h" #include "testing/gtest/include/gtest/gtest.h" +using ::testing::_; +using ::testing::AllOf; +using ::testing::Eq; +using ::testing::Invoke; +using ::testing::IsSupersetOf; +using ::testing::Pair; +using ::testing::Property; +using ::testing::StartsWith; +using ::testing::StrEq; + namespace reporting { + namespace { -class FileUploadImplTest : public ::testing::Test { - protected: - base::test::TaskEnvironment task_environment_{ - base::test::TaskEnvironment::TimeSource::MOCK_TIME}; +constexpr char kUploadPath[] = "/upload"; +constexpr char kRobotAccountId[] = "robot@gmail.com"; +constexpr size_t kDataGranularity = 10; +constexpr size_t kMaxUploadBufferSize = kDataGranularity * 2; +constexpr char kUploadId[] = "ABC"; +constexpr char kResumableUrl[] = + "/upload?upload_id=ABC&upload_protocol=resumable"; +constexpr char kTokenInvalid[] = "INVALID_TOKEN"; +constexpr char kTokenValid[] = "VALID_TOKEN"; + +constexpr char kTestData[] = + "0123456789012345678901234567890123456789012345678901234567890123456789"; +constexpr size_t kTestDataSize = sizeof(kTestData) - 1; + +// Test-only access token manager fake, that allows to pre-populate +// expected valid and invalid tokens ahead of the test execution. +class FakeOAuth2AccessTokenManagerWithCaching + : public FakeOAuth2AccessTokenManager { + public: + explicit FakeOAuth2AccessTokenManagerWithCaching( + OAuth2AccessTokenManager::Delegate* delegate); + + FakeOAuth2AccessTokenManagerWithCaching( + const FakeOAuth2AccessTokenManagerWithCaching&) = delete; + FakeOAuth2AccessTokenManagerWithCaching& operator=( + const FakeOAuth2AccessTokenManagerWithCaching&) = delete; + + ~FakeOAuth2AccessTokenManagerWithCaching() override; + + // FakeOAuth2AccessTokenManager: + void FetchOAuth2Token( + OAuth2AccessTokenManager::RequestImpl* request, + const CoreAccountId& account_id, + scoped_refptr<::network::SharedURLLoaderFactory> url_loader_factory, + const std::string& client_id, + const std::string& client_secret, + const std::string& consumer_name, + const OAuth2AccessTokenManager::ScopeSet& scopes) override; + void InvalidateAccessTokenImpl( + const CoreAccountId& account_id, + const std::string& client_id, + const OAuth2AccessTokenManager::ScopeSet& scopes, + const std::string& access_token) override; + + void AddTokenToQueue(const std::string& token); + bool IsTokenValid(const std::string& token) const; + void SetTokenValid(const std::string& token); + void SetTokenInvalid(const std::string& token); - FileUploadJob::TestEnvironment manager_test_env_; + private: + base::queue token_replies_; + std::set valid_tokens_; }; -TEST_F(FileUploadImplTest, Dummy) { - EXPECT_FALSE(false); +FakeOAuth2AccessTokenManagerWithCaching:: + FakeOAuth2AccessTokenManagerWithCaching( + OAuth2AccessTokenManager::Delegate* delegate) + : FakeOAuth2AccessTokenManager(delegate) {} + +FakeOAuth2AccessTokenManagerWithCaching:: + ~FakeOAuth2AccessTokenManagerWithCaching() = default; + +void FakeOAuth2AccessTokenManagerWithCaching::FetchOAuth2Token( + OAuth2AccessTokenManager::RequestImpl* request, + const CoreAccountId& account_id, + scoped_refptr<::network::SharedURLLoaderFactory> url_loader_factory, + const std::string& client_id, + const std::string& client_secret, + const std::string& consumer_name, + const OAuth2AccessTokenManager::ScopeSet& scopes) { + GoogleServiceAuthError response_error = + GoogleServiceAuthError::AuthErrorNone(); + OAuth2AccessTokenConsumer::TokenResponse token_response; + if (token_replies_.empty()) { + response_error = + GoogleServiceAuthError::FromServiceError("Service unavailable."); + } else { + token_response.access_token = token_replies_.front(); + token_response.expiration_time = base::Time::Now(); + token_replies_.pop(); + } + base::SingleThreadTaskRunner::GetCurrentDefault()->PostTask( + FROM_HERE, + base::BindOnce(&OAuth2AccessTokenManager::RequestImpl::InformConsumer, + request->AsWeakPtr(), response_error, token_response)); +} + +void FakeOAuth2AccessTokenManagerWithCaching::AddTokenToQueue( + const std::string& token) { + token_replies_.push(token); +} + +bool FakeOAuth2AccessTokenManagerWithCaching::IsTokenValid( + const std::string& token) const { + return valid_tokens_.find(token) != valid_tokens_.end(); +} + +void FakeOAuth2AccessTokenManagerWithCaching::SetTokenValid( + const std::string& token) { + valid_tokens_.insert(token); +} + +void FakeOAuth2AccessTokenManagerWithCaching::SetTokenInvalid( + const std::string& token) { + valid_tokens_.erase(token); +} + +void FakeOAuth2AccessTokenManagerWithCaching::InvalidateAccessTokenImpl( + const CoreAccountId& account_id, + const std::string& client_id, + const OAuth2AccessTokenManager::ScopeSet& scopes, + const std::string& access_token) { + SetTokenInvalid(access_token); } + +class FakeOAuth2AccessTokenManagerDelegate + : public OAuth2AccessTokenManager::Delegate { + public: + FakeOAuth2AccessTokenManagerDelegate() = default; + ~FakeOAuth2AccessTokenManagerDelegate() override = default; + + // OAuth2AccessTokenManager::Delegate: + std::unique_ptr CreateAccessTokenFetcher( + const CoreAccountId& account_id, + scoped_refptr<::network::SharedURLLoaderFactory> url_loader_factory, + OAuth2AccessTokenConsumer* consumer) override { + EXPECT_EQ(CoreAccountId(kRobotAccountId), account_id); + return GaiaAccessTokenFetcher:: + CreateExchangeRefreshTokenForAccessTokenInstance( + consumer, url_loader_factory, "fake_refresh_token"); + } + + bool HasRefreshToken(const CoreAccountId& account_id) const override { + return CoreAccountId(kRobotAccountId) == account_id; + } +}; + } // namespace + +class FileUploadDelegateTest : public ::testing::Test { + protected: + FileUploadDelegateTest() { DETACH_FROM_SEQUENCE(sequence_checker_); } + + const GURL GetServerURL(base::StringPiece relative_path) const { + return test_server_.GetURL(relative_path); + } + + void SetUp() override { + url_loader_factory_ = + base::MakeRefCounted<::network::TestSharedURLLoaderFactory>(); + test_server_.RegisterRequestHandler(base::BindRepeating( + &FileUploadDelegateTest::HandlePostRequest, base::Unretained(this))); + ASSERT_TRUE(test_server_.Start()); + PrepareFileForUpload(); + } + + void TearDown() override { + ASSERT_TRUE(test_server_.ShutdownAndWaitUntilComplete()); + } + + std::unique_ptr PrepareFileUploadDelegate() { + auto delegate = std::make_unique(); + DCHECK_CALLED_ON_VALID_SEQUENCE(delegate->sequence_checker_); + delegate->upload_url_ = GetServerURL(kUploadPath); + delegate->account_id_ = CoreAccountId(kRobotAccountId); + delegate->access_token_manager_ = &access_token_manager_; + delegate->url_loader_factory_ = url_loader_factory_; + delegate->traffic_annotation_ = + std::make_unique<::net::NetworkTrafficAnnotationTag>( + TRAFFIC_ANNOTATION_FOR_TESTS); + delegate->max_upload_buffer_size_ = 20; + return delegate; + } + + void PrepareFileForUpload() { + ASSERT_TRUE(temp_dir_.CreateUniqueTempDir()); + origin_path_ = temp_dir_.GetPath().AppendASCII("upload_file"); + base::File file(origin_path_, + base::File::FLAG_CREATE | base::File::FLAG_WRITE); + ASSERT_TRUE(file.IsValid()); + ASSERT_THAT(file.error_details(), Eq(base::File::FILE_OK)); + + const int bytes_written = file.Write(0, kTestData, kTestDataSize); + EXPECT_THAT(bytes_written, Eq(static_cast(kTestDataSize))); + } + + std::unique_ptr<::net::test_server::HttpResponse> HandlePostRequest( + const ::net::test_server::HttpRequest& request) { + auto response = std::make_unique<::net::test_server::BasicHttpResponse>(); + DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_); + mock_request_call_.Call(request, response.get()); + return std::move(response); + } + + void ExpectStart(const ::net::test_server::HttpRequest& request) { + EXPECT_THAT(request.method, Eq(::net::test_server::METHOD_POST)); + EXPECT_THAT(request.headers, + IsSupersetOf({Pair("Authorization", StartsWith("Bearer "))})); + EXPECT_THAT( + request.headers, + IsSupersetOf({ + Pair("Authorization", + ::testing::MatcherCast(StartsWith("Bearer "))), + Pair("X-Goog-Upload-Protocol", + ::testing::MatcherCast(StrEq("resumable"))), + Pair("X-Goog-Upload-Command", + ::testing::MatcherCast(StrEq("start"))), + Pair("X-Goog-Upload-Header-Content-Length", + ::testing::MatcherCast( + StrEq(base::NumberToString(kTestDataSize)))), + Pair("X-Goog-Upload-Header-Content-Type", + ::testing::MatcherCast( + StrEq("application/octet-stream"))), + })); + } + + void ExpectQuery(const ::net::test_server::HttpRequest& request) { + EXPECT_THAT(request.method, Eq(::net::test_server::METHOD_POST)); + EXPECT_THAT(request.relative_url, StrEq(kResumableUrl)); + EXPECT_THAT(request.headers, + IsSupersetOf({ + Pair("X-Goog-Upload-Protocol", StrEq("resumable")), + Pair("X-Goog-Upload-Command", StrEq("query")), + })); + } + + void ExpectStep(size_t offset, + const ::net::test_server::HttpRequest& request) { + EXPECT_THAT(request.method, Eq(::net::test_server::METHOD_POST)); + EXPECT_THAT(request.relative_url, StrEq(kResumableUrl)); + EXPECT_THAT( + request.headers, + IsSupersetOf({ + Pair("X-Goog-Upload-Protocol", StrEq("resumable")), + Pair("X-Goog-Upload-Command", StrEq("upload")), + Pair("X-Goog-Upload-Offset", StrEq(base::NumberToString(offset))), + })); + } + + void ExpectFinish(const ::net::test_server::HttpRequest& request) { + EXPECT_THAT(request.method, Eq(::net::test_server::METHOD_POST)); + EXPECT_THAT(request.relative_url, StrEq(kResumableUrl)); + EXPECT_THAT(request.headers, + IsSupersetOf({ + Pair("X-Goog-Upload-Protocol", StrEq("resumable")), + Pair("X-Goog-Upload-Command", StrEq("finalize")), + })); + } + + void EnsureOriginFileIsErased() { + task_environment_.RunUntilIdle(); // Let file deletion finish. + EXPECT_FALSE(base::PathExists(origin_path_)); + } + std::string origin_path() const { return origin_path_.MaybeAsASCII(); } + + content::BrowserTaskEnvironment task_environment_{ + content::BrowserTaskEnvironment::MainThreadType::IO}; + + // Make sure `mock_request_call_` is called sequentially. + SEQUENCE_CHECKER(sequence_checker_); + ::testing::MockFunction + mock_request_call_; + + FakeOAuth2AccessTokenManagerWithCaching access_token_manager_{ + &token_manager_delegate_}; + + private: + base::ScopedTempDir temp_dir_; + base::FilePath origin_path_; + ::net::EmbeddedTestServer test_server_; + scoped_refptr<::network::TestSharedURLLoaderFactory> url_loader_factory_; + FakeOAuth2AccessTokenManagerDelegate token_manager_delegate_; +}; + +TEST_F(FileUploadDelegateTest, SuccessfulUploadStart) { + // Prepare the delegate. + std::unique_ptr delegate = + PrepareFileUploadDelegate(); + + // Prepare access token. + access_token_manager_.SetTokenValid(kTokenValid); + access_token_manager_.AddTokenToQueue(kTokenValid); + + // Set up responses. + EXPECT_CALL(mock_request_call_, Call(_, _)) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStart(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->AddCustomHeader("X-Goog-Upload-Url", + GetServerURL(kResumableUrl).spec()); + response->set_code(::net::HTTP_OK); + })); + + test::TestEvent< + StatusOr>> + init_done; + delegate->DoInitiate(origin_path(), GetServerURL(kResumableUrl).spec(), + init_done.cb()); + const auto& result = init_done.result(); + ASSERT_OK(result) << result.status(); + ASSERT_THAT(result.ValueOrDie().first, + Eq(static_cast(kTestDataSize))); + ASSERT_THAT(result.ValueOrDie().second, + StrEq(base::StrCat( + {origin_path(), "\n", GetServerURL(kResumableUrl).spec()}))); +} + +TEST_F(FileUploadDelegateTest, FailedUploadStart) { + // Prepare the delegate. + std::unique_ptr delegate = + PrepareFileUploadDelegate(); + + // Set up responses. + EXPECT_CALL(mock_request_call_, Call(_, _)) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStart(request); + response->AddCustomHeader("X-Goog-Upload-Status", "final"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->AddCustomHeader("X-Goog-Upload-Url", + GetServerURL(kResumableUrl).spec()); + response->set_code(::net::HTTP_OK); + })) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStart(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Url", + GetServerURL(kResumableUrl).spec()); + response->set_code(::net::HTTP_OK); + })) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStart(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->set_code(::net::HTTP_OK); + })) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStart(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->AddCustomHeader("X-Goog-Upload-Url", + GetServerURL(kResumableUrl).spec()); + response->set_code(::net::HTTP_INTERNAL_SERVER_ERROR); + })); + + access_token_manager_.SetTokenValid(kTokenValid); + { + // Prepare access token. + access_token_manager_.AddTokenToQueue(kTokenValid); + + test::TestEvent< + StatusOr>> + init_done; + delegate->DoInitiate(origin_path(), GetServerURL(kResumableUrl).spec(), + init_done.cb()); + EXPECT_THAT(init_done.result().status(), + AllOf(Property(&Status::error_code, Eq(error::DATA_LOSS)), + Property(&Status::error_message, + StrEq("Unexpected upload status=final")))); + } + { + // Prepare access token. + access_token_manager_.AddTokenToQueue(kTokenValid); + + test::TestEvent< + StatusOr>> + init_done; + delegate->DoInitiate(origin_path(), GetServerURL(kResumableUrl).spec(), + init_done.cb()); + EXPECT_THAT(init_done.result().status(), + AllOf(Property(&Status::error_code, Eq(error::DATA_LOSS)), + Property(&Status::error_message, + StrEq("Unexpected upload granularity=")))); + } + { + // Prepare access token. + access_token_manager_.AddTokenToQueue(kTokenValid); + + test::TestEvent< + StatusOr>> + init_done; + delegate->DoInitiate(origin_path(), GetServerURL(kResumableUrl).spec(), + init_done.cb()); + EXPECT_THAT(init_done.result().status(), + AllOf(Property(&Status::error_code, Eq(error::DATA_LOSS)), + Property(&Status::error_message, + StrEq("No upload URL returned")))); + } + { + // Prepare access token. + access_token_manager_.AddTokenToQueue(kTokenValid); + + test::TestEvent< + StatusOr>> + init_done; + delegate->DoInitiate(origin_path(), GetServerURL(kResumableUrl).spec(), + init_done.cb()); + EXPECT_THAT( + init_done.result().status(), + AllOf( + Property(&Status::error_code, Eq(error::DATA_LOSS)), + Property(&Status::error_message, + StrEq("POST request failed with HTTP status code 500")))); + } + + access_token_manager_.SetTokenInvalid(kTokenInvalid); + { + access_token_manager_.SetTokenValid(kTokenInvalid); + + test::TestEvent< + StatusOr>> + init_done; + delegate->DoInitiate(origin_path(), GetServerURL(kResumableUrl).spec(), + init_done.cb()); + EXPECT_THAT( + init_done.result().status(), + AllOf( + Property(&Status::error_code, Eq(error::UNAUTHENTICATED)), + Property( + &Status::error_message, + StrEq( + "Service responded with error: 'Service unavailable.'")))); + } +} + +TEST_F(FileUploadDelegateTest, SuccessfulUploadStep) { + // Prepare the delegate. + std::unique_ptr delegate = + PrepareFileUploadDelegate(); + + // Set up responses: query at offset=2*granularity, and make one upload. + EXPECT_CALL(mock_request_call_, Call(_, _)) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectQuery(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->AddCustomHeader("X-Goog-Upload-Size-Received", + base::NumberToString(kMaxUploadBufferSize)); + response->set_code(::net::HTTP_OK); + })) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStep(kMaxUploadBufferSize, request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->set_code(::net::HTTP_OK); + })); + + test::TestEvent< + StatusOr>> + step_done; + delegate->DoNextStep( + kTestDataSize, kMaxUploadBufferSize, + base::StrCat({origin_path(), "\n", GetServerURL(kResumableUrl).spec()}), + step_done.cb()); + const auto& result = step_done.result(); + ASSERT_OK(result) << result.status(); + ASSERT_THAT( + result.ValueOrDie().first, + Eq(static_cast(kMaxUploadBufferSize + kMaxUploadBufferSize))); + ASSERT_THAT(result.ValueOrDie().second, + StrEq(base::StrCat( + {origin_path(), "\n", GetServerURL(kResumableUrl).spec()}))); +} + +TEST_F(FileUploadDelegateTest, SuccessfulUploadStepTillEnd) { + // Prepare the delegate. + std::unique_ptr delegate = + PrepareFileUploadDelegate(); + + // Set up responses: query at offset=2*granularity, and make one upload. + EXPECT_CALL(mock_request_call_, Call(_, _)) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectQuery(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->AddCustomHeader( + "X-Goog-Upload-Size-Received", + base::NumberToString(kTestDataSize - kMaxUploadBufferSize)); + response->set_code(::net::HTTP_OK); + })) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectStep(kTestDataSize - kMaxUploadBufferSize, request); + response->AddCustomHeader("X-Goog-Upload-Status", "final"); + response->set_code(::net::HTTP_OK); + })); + + test::TestEvent< + StatusOr>> + step_done; + delegate->DoNextStep( + kTestDataSize, kTestDataSize - kMaxUploadBufferSize, + base::StrCat({origin_path(), "\n", GetServerURL(kResumableUrl).spec()}), + step_done.cb()); + const auto& result = step_done.result(); + ASSERT_OK(result) << result.status(); + ASSERT_THAT(result.ValueOrDie().first, + Eq(static_cast(kTestDataSize))); + ASSERT_THAT(result.ValueOrDie().second, + StrEq(base::StrCat( + {origin_path(), "\n", GetServerURL(kResumableUrl).spec()}))); +} + +// TODO(b/264399295): Add failure tests. + +TEST_F(FileUploadDelegateTest, SuccessfulUploadFinish) { + // Prepare the delegate. + std::unique_ptr delegate = + PrepareFileUploadDelegate(); + + // Set up responses: query at offset=total, and finalize. + EXPECT_CALL(mock_request_call_, Call(_, _)) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectQuery(request); + response->AddCustomHeader("X-Goog-Upload-Status", "active"); + response->AddCustomHeader("X-Goog-Upload-Chunk-Granularity", + base::NumberToString(kDataGranularity)); + response->AddCustomHeader("X-Goog-Upload-Size-Received", + base::NumberToString(kTestDataSize)); + response->set_code(::net::HTTP_OK); + })) + .WillOnce(Invoke([this](const ::net::test_server::HttpRequest& request, + ::net::test_server::BasicHttpResponse* response) { + ExpectFinish(request); + response->AddCustomHeader("X-Goog-Upload-Status", "final"); + response->AddCustomHeader("X-Goog-Upload-Size-Received", + base::NumberToString(kTestDataSize)); + response->AddCustomHeader("X-GUploader-UploadID", kUploadId); + response->set_code(::net::HTTP_OK); + })); + + test::TestEvent> finish_done; + delegate->DoFinalize( + base::StrCat({origin_path(), "\n", GetServerURL(kResumableUrl).spec()}), + finish_done.cb()); + const auto& result = finish_done.result(); + ASSERT_OK(result) << result.status(); + ASSERT_THAT(result.ValueOrDie(), + StrEq(base::StrCat({"Upload_id=", kUploadId}))); + + EnsureOriginFileIsErased(); +} + +// TODO(b/264399295): Add failure tests. } // namespace reporting diff --git a/chrome/browser/policy/messaging_layer/upload/file_upload_job.h b/chrome/browser/policy/messaging_layer/upload/file_upload_job.h index 3d165d28314e4..e016cfb06b6a1 100644 --- a/chrome/browser/policy/messaging_layer/upload/file_upload_job.h +++ b/chrome/browser/policy/messaging_layer/upload/file_upload_job.h @@ -71,7 +71,7 @@ class FileUploadJob { // Asynchronously finalizes upload (once `uploaded` reached `total`). // Calls back with `access_parameters`, or Status in case of error. virtual void DoFinalize( - base::StringPiece session_token, + base::StringPiece access_parameters, base::OnceCallback)> cb) = 0; diff --git a/chrome/browser/policy/messaging_layer/upload/upload_client.cc b/chrome/browser/policy/messaging_layer/upload/upload_client.cc index ca83962452f0d..66945a9e390e0 100644 --- a/chrome/browser/policy/messaging_layer/upload/upload_client.cc +++ b/chrome/browser/policy/messaging_layer/upload/upload_client.cc @@ -6,6 +6,8 @@ #include "base/memory/ptr_util.h" #include "base/task/sequenced_task_runner.h" +#include "build/build_config.h" +#include "build/chromeos_buildflags.h" #include "chrome/browser/policy/messaging_layer/upload/dm_server_uploader.h" #include "chrome/browser/policy/messaging_layer/upload/file_upload_impl.h" #include "chrome/browser/policy/messaging_layer/upload/record_handler_impl.h" @@ -20,6 +22,18 @@ namespace reporting { +namespace { + +std::unique_ptr CreateFileUploadDelegate() { +#if BUILDFLAG(IS_CHROMEOS_ASH) + return std::make_unique(); +#else // !BUILDFLAG(IS_CHROMEOS_ASH) + // No file uploads for all other configurations. + return nullptr; +#endif // BUILDFLAG(IS_CHROMEOS_ASH) +} +} // namespace + // static void UploadClient::Create(CreatedCallback created_cb) { std::move(created_cb).Run(base::WrapUnique(new UploadClient())); @@ -48,9 +62,9 @@ Status UploadClient::EnqueueUpload( UploadClient::UploadClient() : sequenced_task_runner_(base::SequencedTaskRunner::GetCurrentDefault()), - handler_(std::make_unique( - sequenced_task_runner_, - std::make_unique())) {} + handler_( + std::make_unique(sequenced_task_runner_, + CreateFileUploadDelegate())) {} UploadClient::~UploadClient() = default; diff --git a/chrome/browser/policy/messaging_layer/upload/upload_client_unittest.cc b/chrome/browser/policy/messaging_layer/upload/upload_client_unittest.cc index f4872652195d4..6de2f1b7ce23a 100644 --- a/chrome/browser/policy/messaging_layer/upload/upload_client_unittest.cc +++ b/chrome/browser/policy/messaging_layer/upload/upload_client_unittest.cc @@ -37,6 +37,7 @@ #include "chrome/browser/ash/login/users/fake_chrome_user_manager.h" #include "chrome/test/base/testing_profile.h" #include "components/user_manager/scoped_user_manager.h" +#include "google_apis/gaia/core_account_id.h" #endif // BUILDFLAG(IS_CHROMEOS_ASH) namespace reporting { diff --git a/chrome/test/BUILD.gn b/chrome/test/BUILD.gn index d61de0cb916ae..a9df65e0aa57c 100644 --- a/chrome/test/BUILD.gn +++ b/chrome/test/BUILD.gn @@ -5628,7 +5628,6 @@ test("unit_tests") { "../browser/policy/messaging_layer/upload/dm_server_uploader_unittest.cc", "../browser/policy/messaging_layer/upload/encrypted_reporting_client_unittest.cc", "../browser/policy/messaging_layer/upload/event_upload_size_controller_unittest.cc", - "../browser/policy/messaging_layer/upload/file_upload_impl_unittest.cc", "../browser/policy/messaging_layer/upload/file_upload_job_unittest.cc", "../browser/policy/messaging_layer/upload/network_condition_service_unittest.cc", "../browser/policy/messaging_layer/upload/record_handler_impl_unittest.cc", @@ -5851,6 +5850,7 @@ test("unit_tests") { if (is_chromeos_ash) { sources += [ "../browser/metrics/chrome_metrics_service_client_ash_unittest.cc", + "../browser/policy/messaging_layer/upload/file_upload_impl_unittest.cc", "../browser/ui/ash/arc_vm_data_migration_confirmation_dialog_unittest.cc", ] } diff --git a/tools/traffic_annotation/summary/annotations.xml b/tools/traffic_annotation/summary/annotations.xml index ef4fb56737765..3f33ec8dc3baa 100644 --- a/tools/traffic_annotation/summary/annotations.xml +++ b/tools/traffic_annotation/summary/annotations.xml @@ -419,4 +419,5 @@ Refer to README.md for content description and update process. + diff --git a/tools/traffic_annotation/summary/grouping.xml b/tools/traffic_annotation/summary/grouping.xml index d2a3d21ed941d..c4835075558f3 100644 --- a/tools/traffic_annotation/summary/grouping.xml +++ b/tools/traffic_annotation/summary/grouping.xml @@ -284,6 +284,7 @@ after discussions on the right group. +