Skip to content

Update crt fix chunked #3348

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Mar 19, 2025
Merged

Update crt fix chunked #3348

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dcef5d8
Update CRT to v0.31.1
sbiscigl Mar 18, 2025
87121b4
fix CRT HTTP client for aws chunked requests
sbiscigl Mar 18, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion crt/aws-crt-cpp
Open in desktop
Submodule aws-crt-cpp updated 11 files
+3 −5 .github/workflows/ci.yml
+1 −1 crt/aws-c-auth
+1 −1 crt/aws-c-cal
+1 −1 crt/aws-c-common
+1 −1 crt/aws-c-event-stream
+1 −1 crt/aws-c-http
+1 −1 crt/aws-c-io
+1 −1 crt/aws-c-s3
+1 −1 crt/aws-lc
+1 −1 crt/s2n
+2 −6 source/Types.cpp
20 changes: 10 additions & 10 deletions prefetch_crt_dependency.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,21 @@
# SPDX-License-Identifier: Apache-2.0.

CRT_URI_PREFIX=https://codeload.github.com/awslabs
CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/c776ab5b38036662ebd0569d2483b98a176f9819 # v0.31.0
CRT_URI=${CRT_URI_PREFIX}/aws-crt-cpp/zip/0a27f7412660aeb2cbb0e8c2f08fdf2d8768c918 # v0.31.1

AWS_C_AUTH_URI=${CRT_URI_PREFIX}/aws-c-auth/zip/2d85beff96bee7ee4734c21c7fc6b15e9d07b85e # v0.8.5
AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/7299c6ab9244595b140d604475cdd6c6921be8ae # v0.8.3
AWS_C_COMMON_URI=${CRT_URI_PREFIX}/aws-c-common/zip/6401c830ffcd82ee9c9e26255f2fadf7092c7321 # v0.11.1
AWS_C_AUTH_URI=${CRT_URI_PREFIX}/aws-c-auth/zip/01dd06acd2b8865a4a6bc232380ee69a042af47d # v0.8.6
AWS_C_CAL_URI=${CRT_URI_PREFIX}/aws-c-cal/zip/d59c198db17c42a48e3ee105d12357f5a9efecf3 # v0.8.7
AWS_C_COMMON_URI=${CRT_URI_PREFIX}/aws-c-common/zip/7fb0071ab88182bffcc18a4a09bdb4dd2a5751d8 # v0.12.0
AWS_C_COMPRESSION_URI=${CRT_URI_PREFIX}/aws-c-compression/zip/f951ab2b819fc6993b6e5e6cfef64b1a1554bfc8 # v0.3.1
AWS_C_EVENT_STREAM_URI=${CRT_URI_PREFIX}/aws-c-event-stream/zip/4bd476bd0c629e8fab4ec0ace92830efc6a79e6c # v0.5.2
AWS_C_HTTP_URI=${CRT_URI_PREFIX}/aws-c-http/zip/590c7b597f87e5edc080b8b77418690c30319832 # v0.9.3
AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/5fcecfc621059e254f2dc0dcac46265fcba0bf3a # v0.16.0
AWS_C_EVENT_STREAM_URI=${CRT_URI_PREFIX}/aws-c-event-stream/zip/9312b052583183b98526aaeb91e5c72ec3db9627 # v0.5.4
AWS_C_HTTP_URI=${CRT_URI_PREFIX}/aws-c-http/zip/e3a9cabc664630120df25c28ec710199b8e8b15b # v0.9.5
AWS_C_IO_URI=${CRT_URI_PREFIX}/aws-c-io/zip/318f7e57e7871e5b0d48a281cc5dcb7f79ccecdd # v0.17.0
AWS_C_MQTT_URI=${CRT_URI_PREFIX}/aws-c-mqtt/zip/f0cc34cb6f54e050275e3c859594c62776d46d83 # v0.12.2
AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/6eb8be530b100fed5c6d24ca48a57ee2e6098fbf # v0.7.11
AWS_C_S3_URI=${CRT_URI_PREFIX}/aws-c-s3/zip/169842b7e2f81d71d0719d4a77f9c3e186512f99 # v0.7.13
AWS_C_SDKUTILS_URI=${CRT_URI_PREFIX}/aws-c-sdkutils/zip/ba6a28fab7ed5d7f1b3b1d12eb672088be093824 # v0.2.3
AWS_CHECKSUMS_URI=${CRT_URI_PREFIX}/aws-checksums/zip/fb8bd0b8cff00c8c24a35d601fce1b4c611df6da # v0.2.3
AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/becf5785c131012bb5a64f3da6cdb117ddc0f431 # v1.46.1
S2N_URI=${CRT_URI_PREFIX}/s2n/zip/21cefc1091b3953ef543c9e72b932b6431fadc6e # v1.5.13
AWS_LC_URI=${CRT_URI_PREFIX}/aws-lc/zip/d3e6957b9db2d9c587e77396d7b428139047ec31 # v1.48.4
S2N_URI=${CRT_URI_PREFIX}/s2n/zip/4ed4f1a658b70559ec4a18e91d1319daa14b0610 # v1.5.14


echo "Removing CRT"
Expand Down
16 changes: 14 additions & 2 deletions src/aws-cpp-sdk-core/include/aws/core/utils/stream/AwsChunkedStream.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
#include <aws/core/utils/HashingUtils.h>
#include <aws/core/utils/Outcome.h>
#include <aws/core/utils/crypto/Hash.h>
#include <aws/crt/io/Stream.h>

namespace Aws {
namespace Utils {
Expand All @@ -14,10 +15,13 @@ namespace Stream {
static const size_t AWS_DATA_BUFFER_SIZE = 65536;

template <size_t DataBufferSize = AWS_DATA_BUFFER_SIZE>
class AwsChunkedStream {
class AwsChunkedStream: public Aws::Crt::Io::StdIOStreamInputStream {
public:
AwsChunkedStream(Http::HttpRequest *request, const std::shared_ptr<Aws::IOStream> &stream)
: m_chunkingStream{Aws::MakeShared<StringStream>("AwsChunkedStream")}, m_request(request), m_stream(stream) {
: StdIOStreamInputStream{stream},
m_chunkingStream{Aws::MakeShared<StringStream>("AwsChunkedStream")},
m_request(request),
m_stream(stream) {
assert(m_stream != nullptr);
if (m_stream == nullptr) {
AWS_LOGSTREAM_ERROR("AwsChunkedStream", "stream is null");
Expand Down Expand Up @@ -56,6 +60,14 @@ class AwsChunkedStream {
return static_cast<size_t>(m_chunkingStream->gcount());
}

protected:
bool ReadImpl(Crt::ByteBuf &dstBuffer) noexcept override {
size_t amountToRead = dstBuffer.capacity - dstBuffer.len;
const auto bytesRead = BufferedRead(reinterpret_cast<char *>(dstBuffer.buffer), amountToRead);
dstBuffer.len += bytesRead;;
return true;
};

private:
void writeTrailerToUnderlyingStream() {
Aws::StringStream chunkedTrailerStream;
Expand Down
73 changes: 10 additions & 63 deletions src/aws-cpp-sdk-core/source/http/crt/CRTHttpClient.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,12 @@
*/

#include <aws/core/http/crt/CRTHttpClient.h>
#include <aws/core/http/standard/StandardHttpResponse.h>
#include <aws/core/http/standard/StandardHttpRequest.h>
#include <aws/core/utils/ratelimiter/RateLimiterInterface.h>
#include <aws/core/utils/HashingUtils.h>
#include <aws/core/http/standard/StandardHttpResponse.h>
#include <aws/core/utils/crypto/Hash.h>
#include <aws/core/utils/Outcome.h>
#include <aws/core/utils/logging/LogMacros.h>

#include <aws/core/utils/ratelimiter/RateLimiterInterface.h>
#include <aws/core/utils/stream/AwsChunkedStream.h>
#include <aws/crt/http/HttpConnectionManager.h>
#include <aws/crt/http/HttpRequestResponse.h>

Expand All @@ -30,8 +28,7 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream {
m_rateLimiter(rateLimiter),
m_client(client),
m_currentRequest(request),
m_isStreaming(isStreaming),
m_chunkEnd(false)
m_isStreaming(isStreaming)
{
}

Expand All @@ -44,20 +41,6 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream {
return false;
}

bool isAwsChunked = m_currentRequest.HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) &&
m_currentRequest.GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE;

size_t amountToRead = buffer.capacity - buffer.len;
uint8_t* originalBufferPos = buffer.buffer;

// aws-chunk = hex(chunk-size) + CRLF + chunk-data + CRLF
// Needs to reserve bytes of sizeof(hex(chunk-size)) + sizeof(CRLF) + sizeof(CRLF)
if (isAwsChunked)
{
Aws::String amountToReadHexString = Aws::Utils::StringUtils::ToHexString(amountToRead);
amountToRead -= (amountToReadHexString.size() + 4);
}

// initial check to see if we should avoid reading for the moment.
if (!m_rateLimiter || (m_rateLimiter && m_rateLimiter->ApplyCost(0) == std::chrono::milliseconds(0))) {
size_t currentPos = buffer.len;
Expand Down Expand Up @@ -89,47 +72,8 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream {
return true;
}
}

size_t amountRead = newPos - currentPos;

if (isAwsChunked)
{
// if we have a chunk to wrap, wrap it, be sure to update the running checksum.
if (amountRead > 0)
{
if (m_currentRequest.GetRequestHash().second != nullptr)
{
m_currentRequest.GetRequestHash().second->Update(reinterpret_cast<unsigned char*>(originalBufferPos), amountRead);
}

Aws::String hex = Aws::Utils::StringUtils::ToHexString(amountRead);
// this is safe because of the isAwsChunked branch above.
// I don't see a aws_byte_buf equivalent of memmove. This is lifted from the curl implementation.
memmove(originalBufferPos + hex.size() + 2, originalBufferPos, amountRead);
memmove(originalBufferPos + hex.size() + 2 + amountRead, "\r\n", 2);
memmove(originalBufferPos, hex.c_str(), hex.size());
memmove(originalBufferPos + hex.size(), "\r\n", 2);
amountRead += hex.size() + 4;
}
else if (!m_chunkEnd)
{
// if we didn't read anything, then lets finish up the chunk and send it.
// the reference implementation seems to assume only one chunk is allowed, because the chunkEnd bit is never updated.
// keep that same behavior here.
Aws::StringStream chunkedTrailer;
chunkedTrailer << "0\r\n";
if (m_currentRequest.GetRequestHash().second != nullptr)
{
chunkedTrailer << "x-amz-checksum-" << m_currentRequest.GetRequestHash().first << ":"
<< Aws::Utils::HashingUtils::Base64Encode(m_currentRequest.GetRequestHash().second->GetHash().GetResult()) << "\r\n";
}
chunkedTrailer << "\r\n";
amountRead = chunkedTrailer.str().size();
memcpy(originalBufferPos, chunkedTrailer.str().c_str(), amountRead);
m_chunkEnd = true;
}
buffer.len += amountRead;
}
size_t amountRead = newPos - currentPos;

auto& sentHandler = m_currentRequest.GetDataSentEventHandler();
if (sentHandler)
Expand All @@ -153,7 +97,6 @@ class SDKAdaptingInputStream : public Aws::Crt::Io::StdIOStreamInputStream {
const Aws::Http::HttpClient& m_client;
const Aws::Http::HttpRequest& m_currentRequest;
bool m_isStreaming;
bool m_chunkEnd;
};

// Just a wrapper around a Condition Variable and a mutex, which handles wait and timed waits while protecting
Expand Down Expand Up @@ -430,7 +373,11 @@ namespace Aws
if (request->GetContentBody())
{
bool isStreaming = request->IsEventStreamRequest();
crtRequest->SetBody(Aws::MakeShared<SDKAdaptingInputStream>(CRT_HTTP_CLIENT_TAG, m_configuration.writeRateLimiter, request->GetContentBody(), *this, *request, isStreaming));
if (request->HasHeader(Aws::Http::CONTENT_ENCODING_HEADER) && request->GetHeaderValue(Aws::Http::CONTENT_ENCODING_HEADER) == Aws::Http::AWS_CHUNKED_VALUE) {
crtRequest->SetBody(Aws::MakeShared<Aws::Utils::Stream::AwsChunkedStream<>>(CRT_HTTP_CLIENT_TAG, request.get(), request->GetContentBody()));
} else {
crtRequest->SetBody(Aws::MakeShared<SDKAdaptingInputStream>(CRT_HTTP_CLIENT_TAG, m_configuration.writeRateLimiter, request->GetContentBody(), *this, *request, isStreaming));
}
}

Crt::Http::HttpRequestOptions requestOptions;
Expand Down
Loading