From dcef5d8c56322b0183bcc2bce1574bbd04fa0160 Mon Sep 17 00:00:00 2001 From: sbiscigl Date: Tue, 18 Mar 2025 14:41:53 -0400 Subject: [PATCH 1/2] Update CRT to v0.31.1 --- crt/aws-crt-cpp | 2 +- prefetch_crt_dependency.sh | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/crt/aws-crt-cpp b/crt/aws-crt-cpp index c776ab5b380..0a27f741266 160000 --- a/crt/aws-crt-cpp +++ b/crt/aws-crt-cpp @@ -1 +1 @@ -Subproject commit c776ab5b38036662ebd0569d2483b98a176f9819 +Subproject commit 0a27f7412660aeb2cbb0e8c2f08fdf2d8768c918 diff --git a/prefetch_crt_dependency.sh b/prefetch_crt_dependency.sh index 914829886de..fd141515dec 100755 --- a/prefetch_crt_dependency.sh +++ b/prefetch_crt_dependency.sh @@ -3,21 +3,21 @@ # SPDX-License-Identifier: Apache-2.0. CRT_URI_PREFIX=https://codeload.github.com/awslabs -CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/c776ab5b38036662ebd0569d2483b98a176f9819 # v0.31.0 +CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/0a27f7412660aeb2cbb0e8c2f08fdf2d8768c918 # v0.31.1 -AWS_C_AUTH_URI=${CRT_URI_PREFIX}/aws-c-auth/zip/2d85beff96bee7ee4734c21c7fc6b15e9d07b85e # v0.8.5 -AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/7299c6ab9244595b140d604475cdd6c6921be8ae # v0.8.3 -AWS_C_COMMON_URI=${CRT_URI_PREFIX}/aws-c-common/zip/6401c830ffcd82ee9c9e26255f2fadf7092c7321 # v0.11.1 +AWS_C_AUTH_URI=${CRT_URI_PREFIX}/aws-c-auth/zip/01dd06acd2b8865a4a6bc232380ee69a042af47d # v0.8.6 +AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/d59c198db17c42a48e3ee105d12357f5a9efecf3 # v0.8.7 +AWS_C_COMMON_URI=${CRT_URI_PREFIX}/aws-c-common/zip/7fb0071ab88182bffcc18a4a09bdb4dd2a5751d8 # v0.12.0 AWS_C_COMPRESSION_URI=${CRT_URI_PREFIX}/aws-c-compression/zip/f951ab2b819fc6993b6e5e6cfef64b1a1554bfc8 # v0.3.1 -AWS_C_EVENT_STREAM_URI=${CRT_URI_PREFIX}/aws-c-event-stream/zip/4bd476bd0c629e8fab4ec0ace92830efc6a79e6c # v0.5.2 -AWS_C_HTTP_URI=${CRT_URI_PREFIX}/aws-c-http/zip/590c7b597f87e5edc080b8b77418690c30319832 # v0.9.3 -AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/5fcecfc621059e254f2dc0dcac46265fcba0bf3a # v0.16.0 +AWS_C_EVENT_STREAM_URI=${CRT_URI_PREFIX}/aws-c-event-stream/zip/9312b052583183b98526aaeb91e5c72ec3db9627 # v0.5.4 +AWS_C_HTTP_URI=${CRT_URI_PREFIX}/aws-c-http/zip/e3a9cabc664630120df25c28ec710199b8e8b15b # v0.9.5 +AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/318f7e57e7871e5b0d48a281cc5dcb7f79ccecdd # v0.17.0 AWS_C_MQTT_URI=${CRT_URI_PREFIX}/aws-c-mqtt/zip/f0cc34cb6f54e050275e3c859594c62776d46d83 # v0.12.2 -AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/6eb8be530b100fed5c6d24ca48a57ee2e6098fbf # v0.7.11 +AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/169842b7e2f81d71d0719d4a77f9c3e186512f99 # v0.7.13 AWS_C_SDKUTILS_URI=${CRT_URI_PREFIX}/aws-c-sdkutils/zip/ba6a28fab7ed5d7f1b3b1d12eb672088be093824 # v0.2.3 AWS_CHECKSUMS_URI=${CRT_URI_PREFIX}/aws-checksums/zip/fb8bd0b8cff00c8c24a35d601fce1b4c611df6da # v0.2.3 -AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/becf5785c131012bb5a64f3da6cdb117ddc0f431 # v1.46.1 -S2N_URI=${CRT_URI_PREFIX}/s2n/zip/21cefc1091b3953ef543c9e72b932b6431fadc6e # v1.5.13 +AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/d3e6957b9db2d9c587e77396d7b428139047ec31 # v1.48.4 +S2N_URI=${CRT_URI_PREFIX}/s2n/zip/4ed4f1a658b70559ec4a18e91d1319daa14b0610 # v1.5.14 echo "Removing CRT" From 87121b456cb436fad9f02b73649b583b766675e4 Mon Sep 17 00:00:00 2001 From: sbiscigl Date: Tue, 18 Mar 2025 14:42:14 -0400 Subject: [PATCH 2/2] fix CRT HTTP client for aws chunked requests --- .../aws/core/utils/stream/AwsChunkedStream.h | 16 +++- .../source/http/crt/CRTHttpClient.cpp | 73 +++---------------- 2 files changed, 24 insertions(+), 65 deletions(-) diff --git a/src/aws-cpp-sdk-core/include/aws/core/utils/stream/AwsChunkedStream.h b/src/aws-cpp-sdk-core/include/aws/core/utils/stream/AwsChunkedStream.h index af12d2572fa..e91c909d1f7 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/utils/stream/AwsChunkedStream.h +++ b/src/aws-cpp-sdk-core/include/aws/core/utils/stream/AwsChunkedStream.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace Aws { namespace Utils { @@ -14,10 +15,13 @@ namespace Stream { static const size_t AWS_DATA_BUFFER_SIZE = 65536; template -class AwsChunkedStream { +class AwsChunkedStream: public Aws::Crt::Io::StdIOStreamInputStream { public: AwsChunkedStream(Http::HttpRequest *request, const std::shared_ptr &stream) - : m_chunkingStream{Aws::MakeShared("AwsChunkedStream")}, m_request(request), m_stream(stream) { + : StdIOStreamInputStream{stream}, + m_chunkingStream{Aws::MakeShared("AwsChunkedStream")}, + m_request(request), + m_stream(stream) { assert(m_stream != nullptr); if (m_stream == nullptr) { AWS_LOGSTREAM_ERROR("AwsChunkedStream", "stream is null"); @@ -56,6 +60,14 @@ class AwsChunkedStream { return static_cast(m_chunkingStream->gcount()); } +protected: + bool ReadImpl(Crt::ByteBuf &dstBuffer) noexcept override { + size_t amountToRead = dstBuffer.capacity - dstBuffer.len; + const auto bytesRead = BufferedRead(reinterpret_cast(dstBuffer.buffer), amountToRead); + dstBuffer.len += bytesRead;; + return true; + }; + private: void writeTrailerToUnderlyingStream() { Aws::StringStream chunkedTrailerStream; diff --git a/src/aws-cpp-sdk-core/source/http/crt/CRTHttpClient.cpp b/src/aws-cpp-sdk-core/source/http/crt/CRTHttpClient.cpp index 6f968f862bc..c007cf6d256 100644 --- a/src/aws-cpp-sdk-core/source/http/crt/CRTHttpClient.cpp +++ b/src/aws-cpp-sdk-core/source/http/crt/CRTHttpClient.cpp @@ -4,14 +4,12 @@ */ #include -#include #include -#include -#include +#include #include -#include #include - +#include +#include #include #include @@ -30,8 +28,7 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream { m_rateLimiter(rateLimiter), m_client(client), m_currentRequest(request), - m_isStreaming(isStreaming), - m_chunkEnd(false) + m_isStreaming(isStreaming) { } @@ -44,20 +41,6 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream { return false; } - bool isAwsChunked = m_currentRequest.HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) && - m_currentRequest.GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE; - - size_t amountToRead = buffer.capacity - buffer.len; - uint8_t* originalBufferPos = buffer.buffer; - - // aws-chunk = hex(chunk-size) + CRLF + chunk-data + CRLF - // Needs to reserve bytes of sizeof(hex(chunk-size)) + sizeof(CRLF) + sizeof(CRLF) - if (isAwsChunked) - { - Aws::String amountToReadHexString = Aws::Utils::StringUtils::ToHexString(amountToRead); - amountToRead -= (amountToReadHexString.size() + 4); - } - // initial check to see if we should avoid reading for the moment. if (!m_rateLimiter || (m_rateLimiter && m_rateLimiter->ApplyCost(0) == std::chrono::milliseconds(0))) { size_t currentPos = buffer.len; @@ -89,47 +72,8 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream { return true; } } - - size_t amountRead = newPos - currentPos; - - if (isAwsChunked) - { - // if we have a chunk to wrap, wrap it, be sure to update the running checksum. - if (amountRead > 0) - { - if (m_currentRequest.GetRequestHash().second != nullptr) - { - m_currentRequest.GetRequestHash().second->Update(reinterpret_cast(originalBufferPos), amountRead); - } - Aws::String hex = Aws::Utils::StringUtils::ToHexString(amountRead); - // this is safe because of the isAwsChunked branch above. - // I don't see a aws_byte_buf equivalent of memmove. This is lifted from the curl implementation. - memmove(originalBufferPos + hex.size() + 2, originalBufferPos, amountRead); - memmove(originalBufferPos + hex.size() + 2 + amountRead, "\r\n", 2); - memmove(originalBufferPos, hex.c_str(), hex.size()); - memmove(originalBufferPos + hex.size(), "\r\n", 2); - amountRead += hex.size() + 4; - } - else if (!m_chunkEnd) - { - // if we didn't read anything, then lets finish up the chunk and send it. - // the reference implementation seems to assume only one chunk is allowed, because the chunkEnd bit is never updated. - // keep that same behavior here. - Aws::StringStream chunkedTrailer; - chunkedTrailer << "0\r\n"; - if (m_currentRequest.GetRequestHash().second != nullptr) - { - chunkedTrailer << "x-amz-checksum-" << m_currentRequest.GetRequestHash().first << ":" - << Aws::Utils::HashingUtils::Base64Encode(m_currentRequest.GetRequestHash().second->GetHash().GetResult()) << "\r\n"; - } - chunkedTrailer << "\r\n"; - amountRead = chunkedTrailer.str().size(); - memcpy(originalBufferPos, chunkedTrailer.str().c_str(), amountRead); - m_chunkEnd = true; - } - buffer.len += amountRead; - } + size_t amountRead = newPos - currentPos; auto& sentHandler = m_currentRequest.GetDataSentEventHandler(); if (sentHandler) @@ -153,7 +97,6 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream { const Aws::Http::HttpClient& m_client; const Aws::Http::HttpRequest& m_currentRequest; bool m_isStreaming; - bool m_chunkEnd; }; // Just a wrapper around a Condition Variable and a mutex, which handles wait and timed waits while protecting @@ -430,7 +373,11 @@ namespace Aws if (request->GetContentBody()) { bool isStreaming = request->IsEventStreamRequest(); - crtRequest->SetBody(Aws::MakeShared(CRT_HTTP_CLIENT_TAG, m_configuration.writeRateLimiter, request->GetContentBody(), *this, *request, isStreaming)); + if (request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) && request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE) { + crtRequest->SetBody(Aws::MakeShared>(CRT_HTTP_CLIENT_TAG, request.get(), request->GetContentBody())); + } else { + crtRequest->SetBody(Aws::MakeShared(CRT_HTTP_CLIENT_TAG, m_configuration.writeRateLimiter, request->GetContentBody(), *this, *request, isStreaming)); + } } Crt::Http::HttpRequestOptions requestOptions;