Skip to content

hang issue with at <Aws::Crt::Io::ClientBootstrap>::destroy for IAM role based #3890

Description

@rajeev36

Describe the bug

Tried to perform put object operaion using IRSA

uploading is done but hang at destructor

Tried with 1.11.552 . but getting Access Denied error.
with 1.11.860 getting hang issue during destructor
.
bt:

(gdb) bt
#0  0x00007f97e69b749d in syscall () from /lib64/libc.so.6
#1  0x00007f97e71aadd9 in std::__atomic_futex_unsigned_base::_M_futex_wait_until(unsigned int*, unsigned int, bool, std::chrono::duration<long, std::ratio<1l, 1l> >, std::chrono::duration<long, std::ratio<1l, 1000000000l> >) () from /nz/kit.11.3.1.3/sbin/../lib/libstdc++.so.6
#2  0x00007f97e5cb7706 in std::__atomic_futex_unsigned<2147483648u>::_M_load_and_test_until (this=0x3bc76660, __assumed=0, __operand=1, __equal=true, 
    __mo=std::memory_order_acquire, __has_timeout=false, __s=..., __ns=...)
    at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/atomic_futex.h:102
#3  0x00007f97e5cb6e61 in std::__atomic_futex_unsigned<2147483648u>::_M_load_and_test (this=0x3bc76660, __assumed=0, __operand=1, __equal=true, 
    __mo=std::memory_order_acquire) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/atomic_futex.h:122
#4  0x00007f97e5cb5bb6 in std::__atomic_futex_unsigned<2147483648u>::_M_load_when_equal (__mo=std::memory_order_acquire, __val=1, this=0x3bc76660)
    at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/atomic_futex.h:162
#5  std::__future_base::_State_baseV2::wait (this=0x3bc76650) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/future:337
#6  0x00007f97e5cb6d52 in std::__basic_future<void>::wait (this=0x3bc596f8)
    at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/future:692
#7  0x00007f97e5cb56b1 in Aws::Crt::Io::ClientBootstrap::~ClientBootstrap (this=0x3bc596e0, __in_chrg=<optimized out>)
    at /home/nz/aws/aws-sdk-cpp/crt/aws-crt-cpp/source/io/Bootstrap.cpp:95
#8  0x00007f97e5c3e72c in __gnu_cxx::new_allocator<Aws::Crt::Io::ClientBootstrap>::destroy<Aws::Crt::Io::ClientBootstrap> (this=0x3bc596e0, 
    __p=0x3bc596e0) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/ext/new_allocator.h:153
#9  0x00007f97e5c3e651 in std::allocator_traits<std::allocator<Aws::Crt::Io::ClientBootstrap> >::destroy<Aws::Crt::Io::ClientBootstrap> (__a=..., 
    __p=0x3bc596e0) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/alloc_traits.h:497
#10 0x00007f97e5c3e219 in std::_Sp_counted_ptr_inplace<Aws::Crt::Io::ClientBootstrap, std::allocator<Aws::Crt::Io::ClientBootstrap>, (__gnu_cxx::_Lock_policy)2>::_M_dispose (this=0x3bc596d0) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/shared_ptr_base.h:557
#11 0x00007f97e7f20dca in std::_Sp_counted_base<(__gnu_cxx::_Lock_policy)2>::_M_release (this=0x3bc596d0)
    at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/shared_ptr_base.h:155
#12 0x00007f97e7f1b4d7 in std::__shared_count<(__gnu_cxx::_Lock_policy)2>::~__shared_count (this=0x7f97e6453e28 <Aws::g_defaultClientBootstrap+8>, 
    __in_chrg=<optimized out>) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/shared_ptr_base.h:730
#13 0x00007f97e5c3bb42 in std::__shared_ptr<Aws::Crt::Io::ClientBootstrap, (__gnu_cxx::_Lock_policy)2>::~__shared_ptr (
    this=0x7f97e6453e20 <Aws::g_defaultClientBootstrap>, __in_chrg=<optimized out>)
    at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/shared_ptr_base.h:1169
--Type <RET> for more, q to quit, c to continue without paging--
#14 0x00007f97e5c3bb5e in std::shared_ptr<Aws::Crt::Io::ClientBootstrap>::~shared_ptr (this=0x7f97e6453e20 <Aws::g_defaultClientBootstrap>, 
    __in_chrg=<optimized out>) at /home/nz/nps/main/obj/cc/generic/x86_64-generic-linux-gnu/include/c++/9.2.0/bits/shared_ptr.h:103
#15 0x00007f97e69ce207 in __cxa_finalize () from /lib64/libc.so.6
#16 0x00007f97e5b5f376 in __do_global_dtors_aux () from /nz/kit.11.3.1.3/sbin/../lib/bnr/lib64/libaws-cpp-sdk-core.so
#17 0x00007f97e7f7f020 in __x86_data_cache_size () from /lib64/ld-linux-x86-64.so.2
#18 0x0000000000000000 in ?? ()

in case of if (accessKey.empty()) condition , can see the hang
and with passing the acces key and secret key working without any issue

below is the constructor , destructor and put object implementation

#include <aws/core/Aws.h>
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/utils/logging/LogMacros.h>
#include <aws/core/utils/ResourceManager.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/s3/model/UploadPartRequest.h>
#include <aws/s3/model/CompletedPart.h>
#include <aws/s3/model/CompletedMultipartUpload.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
#include <aws/s3/model/AbortMultipartUploadRequest.h>
#include <aws/s3/model/RestoreObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/RestoreObjectResult.h>
#include <aws/s3/model/HeadObjectResult.h>

#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/core/utils/logging/DefaultLogSystem.h>
#include <aws/core/utils/logging/AWSLogging.h>

S3Interface::S3Interface(const std::string &bucketName,
                         const std::string &accessKey,
                         const std::string &secretId,
                         const std::string &region,
                         const std::string &endpoint,
                         size_t transferMgrBufferSize,
                         const std::string &token,
                         int retryRequestCount,
                         int timeOut,
                         const std::string &logLevel,
                         const std::string &storageClass,
                         const int retainDays,
                         const std::string &tier)
  : m_bucketName(bucketName)
  , m_token(token)
  , m_logLevel(logLevel)
  , m_retainDays(retainDays)
{
    // prepend a '/' to the bucket name
    if (!m_bucketName.empty() && m_bucketName.at(0) != '/') {
        m_bucketName = '/' + m_bucketName;
    }
    Aws::SDKOptions options;

    std::string prefix = "aws_sdk." + std::to_string(getppid()) + ".";
    options.loggingOptions.defaultLogPrefix = prefix.c_str();

    Aws::Utils::Logging::InitializeAWSLogging(Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(AWSLOG_TAG,
		    static_cast<Aws::Utils::Logging::LogLevel>(GetLogLevel_UsingName(m_logLevel)),
		    options.loggingOptions.defaultLogPrefix));

    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Using AWS SDK for C++ version: " AWS_SDK_VERSION_STRING);

    Aws::InitAPI(options);

    Aws::Client::ClientConfiguration clientConfig;
    clientConfig.region = region.c_str();

    if (!endpoint.empty()) {
        clientConfig.endpointOverride = endpoint.c_str();
    }
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Endpoint Set: " << clientConfig.endpointOverride);

    // Set timeout values if provided
    if (timeOut > 0) {
        clientConfig.connectTimeoutMs = timeOut * 1000;
        clientConfig.requestTimeoutMs = timeOut * 1000;
        AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Timeout Set: " << timeOut << " seconds");
    }

    // Disable SSL
    clientConfig.verifySSL = false;

    clientConfig.checksumConfig.requestChecksumCalculation = Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED;

    // Build credentials provider based on what's available
    std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credProvider;

    if (accessKey.empty()) {
        // No keys provided — use IAM role via default credential chain (IRSA)
        credProvider = std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
    } else {
        // Keys provided — use them directly (with or without session token)
        Aws::Auth::AWSCredentials credentials( accessKey.c_str(), secretId.c_str(), m_token.empty() ? "" : m_token.c_str() );
        credProvider = std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(credentials);
    }

    m_s3Client.reset(new Aws::S3::S3Client( credProvider, clientConfig,
                                            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true ));

    transferMgr.setBufferSize(transferMgrBufferSize);
    S3Interface::m_retryRequestCount = retryRequestCount;

    SetStorageClass_UsingName(storageClass);
    SetTier_UsingName(tier);
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Storage Class : " << storageClass << ", " << m_storageClass);
}

/* Destructor
 */
S3Interface::~S3Interface()
{
    transferMgr.clean();
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info;

    Aws::ShutdownAPI(options);
    Aws::Utils::Logging::ShutdownAWSLogging();
}

void
S3Interface::putBufferToS3(const char *buffer, size_t size)
{
    AWS_LOGSTREAM_DEBUG(AWSLOG_TAG, "Entry");
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Object : " << m_currObj);

    Aws::S3::Model::PutObjectRequest object_request;
    object_request.SetBucket(Aws::String(m_bucketName.c_str(), m_bucketName.size()));
    object_request.SetKey(Aws::String(m_currObj.c_str(), m_currObj.size()));
    object_request.SetStorageClass(static_cast<Aws::S3::Model::StorageClass>(m_storageClass));
    AWS_LOGSTREAM_DEBUG(AWSLOG_TAG, "Storage Class: " << m_storageClass);

    std::string allocationTag = "tag_" + m_currObj;
    auto ipStream = Aws::MakeShared<Aws::StringStream>(allocationTag.c_str());
    ipStream->rdbuf()->pubsetbuf((char *) buffer, size);
    object_request.SetBody(ipStream);
    object_request.SetContentLength(size);
    AWS_LOGSTREAM_DEBUG(AWSLOG_TAG, "TagListInRequest : " << m_tagList);
    if (!m_tagList.empty()) {
        object_request.SetTagging(m_tagList);
        m_tagList.clear();
    }
    auto put_object_outcome = m_s3Client->PutObject(object_request);

    if (hasOperationFailed(put_object_outcome, AWSLOG_TAG)) {
        throw put_object_outcome.GetError().GetMessage();
    }

    transferMgr.releaseBuffer(buffer);
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Success, object : " << m_currObj);
}

Regression Issue

  • Select this option if this issue appears to be a regression.

Expected Behavior

hang issue to be resolved in case of IRSA

Current Behavior

gettin hang during destruction in case of IRSA

Reproduction Steps

#include <aws/core/Aws.h>
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/utils/logging/LogMacros.h>
#include <aws/core/utils/ResourceManager.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/PutObjectRequest.h>
#include <aws/s3/model/ListObjectsRequest.h>
#include <aws/s3/model/GetObjectRequest.h>
#include <aws/s3/model/DeleteObjectRequest.h>
#include <aws/s3/model/CreateMultipartUploadRequest.h>
#include <aws/s3/model/UploadPartRequest.h>
#include <aws/s3/model/CompletedPart.h>
#include <aws/s3/model/CompletedMultipartUpload.h>
#include <aws/s3/model/CompleteMultipartUploadRequest.h>
#include <aws/s3/model/AbortMultipartUploadRequest.h>
#include <aws/s3/model/RestoreObjectRequest.h>
#include <aws/s3/model/HeadObjectRequest.h>
#include <aws/s3/model/RestoreObjectResult.h>
#include <aws/s3/model/HeadObjectResult.h>

#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/core/utils/logging/DefaultLogSystem.h>
#include <aws/core/utils/logging/AWSLogging.h>

S3Interface::S3Interface(const std::string &bucketName,
                         const std::string &accessKey,
                         const std::string &secretId,
                         const std::string &region,
                         const std::string &endpoint,
                         size_t transferMgrBufferSize,
                         const std::string &token,
                         int retryRequestCount,
                         int timeOut,
                         const std::string &logLevel,
                         const std::string &storageClass,
                         const int retainDays,
                         const std::string &tier)
  : m_bucketName(bucketName)
  , m_token(token)
  , m_logLevel(logLevel)
  , m_retainDays(retainDays)
{
    // prepend a '/' to the bucket name
    if (!m_bucketName.empty() && m_bucketName.at(0) != '/') {
        m_bucketName = '/' + m_bucketName;
    }
    Aws::SDKOptions options;

    std::string prefix = "aws_sdk." + std::to_string(getppid()) + ".";
    options.loggingOptions.defaultLogPrefix = prefix.c_str();

    Aws::Utils::Logging::InitializeAWSLogging(Aws::MakeShared<Aws::Utils::Logging::DefaultLogSystem>(AWSLOG_TAG,
		    static_cast<Aws::Utils::Logging::LogLevel>(GetLogLevel_UsingName(m_logLevel)),
		    options.loggingOptions.defaultLogPrefix));

    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Using AWS SDK for C++ version: " AWS_SDK_VERSION_STRING);

    Aws::InitAPI(options);

    Aws::Client::ClientConfiguration clientConfig;
    clientConfig.region = region.c_str();

    if (!endpoint.empty()) {
        clientConfig.endpointOverride = endpoint.c_str();
    }
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Endpoint Set: " << clientConfig.endpointOverride);

    // Set timeout values if provided
    if (timeOut > 0) {
        clientConfig.connectTimeoutMs = timeOut * 1000;
        clientConfig.requestTimeoutMs = timeOut * 1000;
        AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Timeout Set: " << timeOut << " seconds");
    }

    // Disable SSL
    clientConfig.verifySSL = false;

    clientConfig.checksumConfig.requestChecksumCalculation = Aws::Client::RequestChecksumCalculation::WHEN_REQUIRED;

    // Build credentials provider based on what's available
    std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credProvider;

    if (accessKey.empty()) {
        // No keys provided — use IAM role via default credential chain (IRSA)
        credProvider = std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
    } else {
        // Keys provided — use them directly (with or without session token)
        Aws::Auth::AWSCredentials credentials( accessKey.c_str(), secretId.c_str(), m_token.empty() ? "" : m_token.c_str() );
        credProvider = std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(credentials);
    }

    m_s3Client.reset(new Aws::S3::S3Client( credProvider, clientConfig,
                                            Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true ));

    transferMgr.setBufferSize(transferMgrBufferSize);
    S3Interface::m_retryRequestCount = retryRequestCount;

    SetStorageClass_UsingName(storageClass);
    SetTier_UsingName(tier);
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Storage Class : " << storageClass << ", " << m_storageClass);
}

/* Destructor
 */
S3Interface::~S3Interface()
{
    transferMgr.clean();
    Aws::SDKOptions options;
    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Info;

    Aws::ShutdownAPI(options);
    Aws::Utils::Logging::ShutdownAWSLogging();
}

void
S3Interface::putBufferToS3(const char *buffer, size_t size)
{
    AWS_LOGSTREAM_DEBUG(AWSLOG_TAG, "Entry");
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Object : " << m_currObj);

    Aws::S3::Model::PutObjectRequest object_request;
    object_request.SetBucket(Aws::String(m_bucketName.c_str(), m_bucketName.size()));
    object_request.SetKey(Aws::String(m_currObj.c_str(), m_currObj.size()));
    object_request.SetStorageClass(static_cast<Aws::S3::Model::StorageClass>(m_storageClass));
    AWS_LOGSTREAM_DEBUG(AWSLOG_TAG, "Storage Class: " << m_storageClass);

    std::string allocationTag = "tag_" + m_currObj;
    auto ipStream = Aws::MakeShared<Aws::StringStream>(allocationTag.c_str());
    ipStream->rdbuf()->pubsetbuf((char *) buffer, size);
    object_request.SetBody(ipStream);
    object_request.SetContentLength(size);
    AWS_LOGSTREAM_DEBUG(AWSLOG_TAG, "TagListInRequest : " << m_tagList);
    if (!m_tagList.empty()) {
        object_request.SetTagging(m_tagList);
        m_tagList.clear();
    }
    auto put_object_outcome = m_s3Client->PutObject(object_request);

    if (hasOperationFailed(put_object_outcome, AWSLOG_TAG)) {
        throw put_object_outcome.GetError().GetMessage();
    }

    transferMgr.releaseBuffer(buffer);
    AWS_LOGSTREAM_INFO(AWSLOG_TAG, "Success, object : " << m_currObj);
}

Possible Solution

No response

Additional Information/Context

No response

AWS CPP SDK version used

1.11.860

Compiler and Version used

(GCC) 9.2.0

Operating System and version

Red Hat Enterprise Linux release 8.10 (Ootpa)

Metadata

Metadata

Assignees

No one assigned

    Labels

    bugThis issue is a bug.

    Type

    No type

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions