From 5c1c4d7cae5aba3ac1365eb685dd6554fe9e2613 Mon Sep 17 00:00:00 2001 From: Mattias Ellert Date: Sat, 28 May 2016 06:11:23 +0200 Subject: [PATCH 01/57] Support older libcurl versions --- src/request.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/request.c b/src/request.c index dd23f46..26cad35 100644 --- a/src/request.c +++ b/src/request.c @@ -1372,7 +1372,11 @@ S3Status request_curl_code_to_status(CURLcode code) return S3StatusConnectionFailed; case CURLE_PARTIAL_FILE: return S3StatusOK; +#if LIBCURL_VERSION_NUM >= 0x071101 /* 7.17.1 */ case CURLE_PEER_FAILED_VERIFICATION: +#else + case CURLE_SSL_PEER_CERTIFICATE: +#endif case CURLE_SSL_CACERT: return S3StatusServerFailedVerification; default: From 2f9a24c748d0a5005418a6e4825b8f81427291b4 Mon Sep 17 00:00:00 2001 From: Sivachandran Date: Thu, 30 Jun 2016 10:17:42 +0530 Subject: [PATCH 02/57] Fix OSX build error due to uint64_t format string incompatibility startByte and byteCount variables are explicitly casted to neutral form to avoid incompatibility in uint64_t format string across platforms. --- src/request.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/request.c b/src/request.c index df871b2..840cb56 100644 --- a/src/request.c +++ b/src/request.c @@ -360,9 +360,9 @@ static S3Status compose_amz_headers(const RequestParams *params, } // If byteCount != 0 then we're just copying a range, add header if (params->byteCount > 0) { - headers_append(1, "x-amz-copy-source-range: bytes=%ld-%ld", - params->startByte, - params->startByte + params->byteCount); + headers_append(1, "x-amz-copy-source-range: bytes=%llu-%llu", + (unsigned long long)params->startByte, + (unsigned long long)params->startByte + params->byteCount); } // And the x-amz-metadata-directive header if (properties) { From 806d01d23d157dda4989c8ffdc1b450c1eeb4a99 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Wed, 6 Jul 2016 08:53:41 -0600 Subject: [PATCH 03/57] fix test script to remove leftover test file; fix exit status of test script to reflect failures --- test/test.sh | 55 +++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 3 deletions(-) diff --git a/test/test.sh b/test/test.sh index 4ebba93..4373941 100755 --- a/test/test.sh +++ b/test/test.sh @@ -28,106 +28,132 @@ if [ -z "$S3_COMMAND" ]; then S3_COMMAND=s3 fi +failures=0 + TEST_BUCKET=${TEST_BUCKET_PREFIX}.testbucket # Create the test bucket echo "$S3_COMMAND create $TEST_BUCKET" $S3_COMMAND create $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # List to find it echo "$S3_COMMAND list | grep $TEST_BUCKET" $S3_COMMAND list | grep $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Test it echo "$S3_COMMAND test $TEST_BUCKET" $S3_COMMAND test $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # List to ensure that it is empty echo "$S3_COMMAND list $TEST_BUCKET" $S3_COMMAND list $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Put some data rm -f seqdata seq 1 10000 > seqdata echo "$S3_COMMAND put $TEST_BUCKET/testkey filename=seqdata noStatus=1" $S3_COMMAND put $TEST_BUCKET/testkey filename=seqdata noStatus=1 +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f testkey # Get the data and make sure that it matches echo "$S3_COMMAND get $TEST_BUCKET/testkey filename=testkey" $S3_COMMAND get $TEST_BUCKET/testkey filename=testkey +failures=$(($failures + (($? == 0) ? 0 : 1))) diff seqdata testkey +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f seqdata testkey # Delete the file echo "$S3_COMMAND delete $TEST_BUCKET/testkey" $S3_COMMAND delete $TEST_BUCKET/testkey +failures=$(($failures + (($? == 0) ? 0 : 1))) # Remove the test bucket echo "$S3_COMMAND delete $TEST_BUCKET" $S3_COMMAND delete $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Make sure it's not there echo "$S3_COMMAND list | grep $TEST_BUCKET" $S3_COMMAND list | grep $TEST_BUCKET +failures=$(($failures + (($? == 1) ? 0 : 1))) # Now create it again echo "$S3_COMMAND create $TEST_BUCKET" $S3_COMMAND create $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) # Put 10 files in it for i in `seq 0 9`; do echo "echo \"Hello\" | $S3_COMMAND put $TEST_BUCKET/key_$i" echo "Hello" | $S3_COMMAND put $TEST_BUCKET/key_$i + failures=$(($failures + (($? == 0) ? 0 : 1))) done # List with all details echo "$S3_COMMAND list $TEST_BUCKET" $S3_COMMAND list $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) COPY_BUCKET=${TEST_BUCKET_PREFIX}.copybucket # Create another test bucket and copy a file into it echo "$S3_COMMAND create $COPY_BUCKET" $S3_COMMAND create $COPY_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) echo <> acl @@ -138,18 +164,22 @@ Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET filename=acl" $S3_COMMAND setacl $TEST_BUCKET filename=acl +failures=$(($failures + (($? == 0) ? 0 : 1))) # Test to make sure that it worked rm -f acl_new echo "$S3_COMMAND getacl $TEST_BUCKET filename=acl_new" $S3_COMMAND getacl $TEST_BUCKET filename=acl_new -diff acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) +diff -B acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f acl acl_new # Get the key acl rm -f acl echo "$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl" $S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl +failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone echo <> acl @@ -160,25 +190,44 @@ Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl" $S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl +failures=$(($failures + (($? == 0) ? 0 : 1))) # Test to make sure that it worked rm -f acl_new echo "$S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl_new" $S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl_new -diff acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) +diff -B acl acl_new +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f acl acl_new # Check multipart file upload (>15MB) dd if=/dev/zero of=mpfile bs=1024k count=30 echo "$S3_COMMAND put $TEST_BUCKET/mpfile filename=mpfile" $S3_COMMAND put $TEST_BUCKET/mpfile filename=mpfile +failures=$(($failures + (($? == 0) ? 0 : 1))) echo "$S3_COMMAND get $TEST_BUCKET/mpfile filename=mpfile.get" $S3_COMMAND get $TEST_BUCKET/mpfile filename=mpfile.get +failures=$(($failures + (($? == 0) ? 0 : 1))) diff mpfile mpfile.get +failures=$(($failures + (($? == 0) ? 0 : 1))) rm -f mpfile mpfile.get -# Remove the test file +# Remove the test files +echo "$S3_COMMAND delete $TEST_BUCKET/mpfile" +$S3_COMMAND delete $TEST_BUCKET/mpfile +failures=$(($failures + (($? == 0) ? 0 : 1))) echo "$S3_COMMAND delete $TEST_BUCKET/aclkey" $S3_COMMAND delete $TEST_BUCKET/aclkey +failures=$(($failures + (($? == 0) ? 0 : 1))) echo "$S3_COMMAND delete $TEST_BUCKET" $S3_COMMAND delete $TEST_BUCKET +failures=$(($failures + (($? == 0) ? 0 : 1))) + +if [ ${failures} = 0 ]; then + echo "all tests completed successfully" +else + echo "tests completed with ${failures} failures" +fi + +exit ${failures} From da1f0fb5e0feb823dfe4195bed86fa2e172680d2 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 29 Sep 2016 14:27:32 -0600 Subject: [PATCH 04/57] got v4 auth working on Mac OS --- src/object.c | 2 +- src/request.c | 524 +++++++++++++++++++++++++++++--------------- src/testsignature.c | 59 +++++ src/util.c | 4 +- 4 files changed, 413 insertions(+), 176 deletions(-) create mode 100644 src/testsignature.c diff --git a/src/object.c b/src/object.c index 445b067..b1e640b 100644 --- a/src/object.c +++ b/src/object.c @@ -338,7 +338,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, // Perform the request request_perform(¶ms, requestContext); } - + // delete object -------------------------------------------------------------- diff --git a/src/request.c b/src/request.c index df871b2..b9e63c5 100644 --- a/src/request.c +++ b/src/request.c @@ -32,11 +32,17 @@ #include "request.h" #include "request_context.h" #include "response_headers_handler.h" -#include "util.h" +#ifdef __APPLE__ +#include +#endif #define USER_AGENT_SIZE 256 #define REQUEST_STACK_SIZE 32 +#define SIGNATURE_SCOPE_SIZE 64 + +//#define SIGNATURE_DEBUG + static int verifyPeer; static char userAgentG[USER_AGENT_SIZE]; @@ -61,15 +67,24 @@ typedef struct RequestComputedValues // Storage for amzHeaders (the +256 is for x-amz-acl and x-amz-date) char amzHeadersRaw[COMPACTED_METADATA_BUFFER_SIZE + 256 + 1]; - // Canonicalized x-amz- headers - string_multibuffer(canonicalizedAmzHeaders, + // Length of populated data in raw buffer + int amzHeadersRawLength; + + // Canonicalized headers for signature + string_multibuffer(canonicalizedSignatureHeaders, COMPACTED_METADATA_BUFFER_SIZE + 256 + 1); + // Delimited list of header names used for signature + char signedHeaders[COMPACTED_METADATA_BUFFER_SIZE]; + // URL-Encoded key char urlEncodedKey[MAX_URLENCODED_KEY_SIZE + 1]; // Canonicalized resource - char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE + 1]; + char canonicalURI[MAX_CANONICALIZED_RESOURCE_SIZE + 1]; + + // Canonical sub-resource & query string + char canonicalQueryString[MAX_CANONICALIZED_RESOURCE_SIZE + 1]; // Cache-Control header (or empty) char cacheControlHeader[128]; @@ -105,16 +120,19 @@ typedef struct RequestComputedValues char rangeHeader[128]; // Authorization header - char authorizationHeader[128]; + char authorizationHeader[256]; // Host header char hostHeader[128]; + + // Hex string of hash of request payload + char payloadHash[CC_SHA256_DIGEST_LENGTH * 2 + 1]; } RequestComputedValues; // Called whenever we detect that the request headers have been completely // processed; which happens either when we get our first read/write callback, -// or the request is finished being procesed. Returns nonzero on success, +// or the request is finished being processed. Returns nonzero on success, // zero on failure. static void request_headers_done(Request *request) { @@ -186,7 +204,7 @@ static size_t curl_read_func(void *ptr, size_t size, size_t nmemb, void *data) if (!request->toS3Callback || !request->toS3CallbackBytesRemaining) { return 0; } - + // Don't tell the callback that we are willing to accept more data than we // really are if (len > request->toS3CallbackBytesRemaining) { @@ -244,6 +262,49 @@ static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, } +static S3Status append_amz_header(RequestComputedValues *values, + int addPrefix, + const char *headerName, + const char *headerValue) +{ + int rawPos = values->amzHeadersRawLength + 1; + values->amzHeaders[values->amzHeadersCount++] = &(values->amzHeadersRaw[rawPos]); + + const char *headerStr = headerName; + if (addPrefix) { + char headerNameWithPrefix[S3_MAX_METADATA_SIZE - sizeof(": v")]; + snprintf(headerNameWithPrefix, sizeof(headerNameWithPrefix), + S3_METADATA_HEADER_NAME_PREFIX "%s", headerName); + headerStr = headerNameWithPrefix; + } + + // Make sure the new header (plus ": " plus string terminator) will fit + // in the buffer. + if ((values->amzHeadersRawLength + strlen(headerStr) + strlen(headerValue) + + 3) >= sizeof(values->amzHeadersRaw)) { + return S3StatusMetaDataHeadersTooLong; + } + + for (unsigned long i = 0; i < strlen(headerStr); i++) { + values->amzHeadersRaw[rawPos++] = tolower(headerStr[i]); + } + + strcat(&(values->amzHeadersRaw[rawPos]), ": "); + rawPos += 2; + + for (unsigned long i = 0; i < strlen(headerValue); i++) { + values->amzHeadersRaw[rawPos++] = headerValue[i]; + } + rawPos--; + + while (isblank(values->amzHeadersRaw[rawPos])) { + rawPos--; + } + values->amzHeadersRaw[++rawPos] = '\0'; + values->amzHeadersRawLength = rawPos; + return S3StatusOK; +} + // This function 'normalizes' all x-amz-meta headers provided in // params->requestHeaders, which means it removes all whitespace from // them such that they all look exactly like this: @@ -255,73 +316,28 @@ static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, // these headers in params->amzHeaders (and also sets params->amzHeadersCount // to be the count of the total number of x-amz- headers thus created). static S3Status compose_amz_headers(const RequestParams *params, + const char *dateStr, RequestComputedValues *values) { const S3PutProperties *properties = params->putProperties; values->amzHeadersCount = 0; - values->amzHeadersRaw[0] = 0; - int len = 0; - - // Append a header to amzHeaders, trimming whitespace from the end. - // Does NOT trim whitespace from the beginning. -#define headers_append(isNewHeader, format, ...) \ - do { \ - if (isNewHeader) { \ - values->amzHeaders[values->amzHeadersCount++] = \ - &(values->amzHeadersRaw[len]); \ - } \ - len += snprintf(&(values->amzHeadersRaw[len]), \ - sizeof(values->amzHeadersRaw) - len, \ - format, __VA_ARGS__); \ - if (len >= (int) sizeof(values->amzHeadersRaw)) { \ - return S3StatusMetaDataHeadersTooLong; \ - } \ - while ((len > 0) && (values->amzHeadersRaw[len - 1] == ' ')) { \ - len--; \ - } \ - values->amzHeadersRaw[len++] = 0; \ - } while (0) - -#define header_name_tolower_copy(str, l) \ - do { \ - values->amzHeaders[values->amzHeadersCount++] = \ - &(values->amzHeadersRaw[len]); \ - if ((len + l) >= (int) sizeof(values->amzHeadersRaw)) { \ - return S3StatusMetaDataHeadersTooLong; \ - } \ - int todo = l; \ - while (todo--) { \ - if ((*(str) >= 'A') && (*(str) <= 'Z')) { \ - values->amzHeadersRaw[len++] = 'a' + (*(str) - 'A'); \ - } \ - else { \ - values->amzHeadersRaw[len++] = *(str); \ - } \ - (str)++; \ - } \ - } while (0) + values->amzHeadersRaw[0] = '\0'; + values->amzHeadersRawLength = 0; // Check and copy in the x-amz-meta headers if (properties) { int i; for (i = 0; i < properties->metaDataCount; i++) { const S3NameValue *property = &(properties->metaData[i]); - char headerName[S3_MAX_METADATA_SIZE - sizeof(": v")]; - int l = snprintf(headerName, sizeof(headerName), - S3_METADATA_HEADER_NAME_PREFIX "%s", - property->name); - char *hn = headerName; - header_name_tolower_copy(hn, l); - // Copy in the value - headers_append(0, ": %s", property->value); + append_amz_header(values, 1, property->name, property->value); } // Add the x-amz-acl header, if necessary const char *cannedAclString; switch (properties->cannedAcl) { case S3CannedAclPrivate: - cannedAclString = 0; + cannedAclString = NULL; break; case S3CannedAclPublicRead: cannedAclString = "public-read"; @@ -334,48 +350,67 @@ static S3Status compose_amz_headers(const RequestParams *params, break; } if (cannedAclString) { - headers_append(1, "x-amz-acl: %s", cannedAclString); + append_amz_header(values, 0, "x-amz-acl", cannedAclString); } // Add the x-amz-server-side-encryption header, if necessary if (properties->useServerSideEncryption) { - headers_append(1, "x-amz-server-side-encryption: %s", "AES256"); + append_amz_header(values, 0, "x-amz-server-side-encryption", + "AES256"); } } // Add the x-amz-date header - time_t now = time(NULL); - char date[64]; - struct tm gmt; - strftime(date, sizeof(date), "%a, %d %b %Y %H:%M:%S GMT", gmtime_r(&now, &gmt)); - headers_append(1, "x-amz-date: %s", date); + append_amz_header(values, 0, "x-amz-date", dateStr); if (params->httpRequestType == HttpRequestTypeCOPY) { // Add the x-amz-copy-source header - if (params->copySourceBucketName && params->copySourceBucketName[0] && - params->copySourceKey && params->copySourceKey[0]) { - headers_append(1, "x-amz-copy-source: /%s/%s", - params->copySourceBucketName, - params->copySourceKey); + if (params->copySourceBucketName && params->copySourceBucketName[0] + && params->copySourceKey && params->copySourceKey[0]) { + char bucketKey[S3_MAX_METADATA_SIZE]; + snprintf(bucketKey, sizeof(bucketKey), "/%s/%s", + params->copySourceBucketName, params->copySourceKey); + append_amz_header(values, 0, "x-amz-copy-source", bucketKey); } // If byteCount != 0 then we're just copying a range, add header if (params->byteCount > 0) { - headers_append(1, "x-amz-copy-source-range: bytes=%ld-%ld", - params->startByte, - params->startByte + params->byteCount); + char byteRange[S3_MAX_METADATA_SIZE]; + snprintf(byteRange, sizeof(byteRange), "bytes=%lld-%lld", + params->startByte, params->startByte + params->byteCount); + append_amz_header(values, 0, "x-amz-copy-source-range", byteRange); } // And the x-amz-metadata-directive header if (properties) { - headers_append(1, "%s", "x-amz-metadata-directive: REPLACE"); + append_amz_header(values, 0, "x-amz-metadata-directive", "REPLACE"); } } // Add the x-amz-security-token header if necessary if (params->bucketContext.securityToken) { - headers_append(1, "x-amz-security-token: %s", - params->bucketContext.securityToken); + append_amz_header(values, 0, "x-amz-security-token", + params->bucketContext.securityToken); + } + + if (params->httpRequestType == HttpRequestTypeGET + || params->httpRequestType == HttpRequestTypeCOPY + || params->httpRequestType == HttpRequestTypeDELETE + || params->httpRequestType == HttpRequestTypeHEAD) { + // empty payload + unsigned char md[CC_SHA256_DIGEST_LENGTH]; + CC_SHA256("", 0, md); + values->payloadHash[0] = '\0'; + for (int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + snprintf(&(values->payloadHash[i * 2]), 3, "%02x", md[i]); + } + } + else { + // TODO: figure out how to manage signed payloads + strcpy(values->payloadHash, "UNSIGNED-PAYLOAD"); } + append_amz_header(values, 0, "x-amz-content-sha256", + values->payloadHash); + return S3StatusOK; } @@ -460,14 +495,34 @@ static S3Status compose_standard_headers(const RequestParams *params, len--; } values->hostHeader[len] = 0; - } else { - values->hostHeader[0] = 0; + } + else if (params->bucketContext.hostName) { + size_t len = snprintf(values->hostHeader, sizeof(values->hostHeader), + "Host: %s", params->bucketContext.hostName); + if (len >= sizeof(values->hostHeader)) { + return S3StatusUriTooLong; + } + while (is_blank(values->hostHeader[len])) { + len--; + } + values->hostHeader[len] = 0; + } + else { + size_t len = snprintf(values->hostHeader, sizeof(values->hostHeader), + "Host: %s", S3_DEFAULT_HOSTNAME); + if (len >= sizeof(values->hostHeader)) { + return S3StatusUriTooLong; + } + while (is_blank(values->hostHeader[len])) { + len--; + } + values->hostHeader[len] = 0; } // Cache-Control do_put_header("Cache-Control: %s", cacheControl, cacheControlHeader, S3StatusBadCacheControl, S3StatusCacheControlTooLong); - + // ContentType do_put_header("Content-Type: %s", contentType, contentTypeHeader, S3StatusBadContentType, S3StatusContentTypeTooLong); @@ -481,12 +536,12 @@ static S3Status compose_standard_headers(const RequestParams *params, contentDispositionFilename, contentDispositionHeader, S3StatusBadContentDispositionFilename, S3StatusContentDispositionFilenameTooLong); - + // ContentEncoding do_put_header("Content-Encoding: %s", contentEncoding, contentEncodingHeader, S3StatusBadContentEncoding, S3StatusContentEncodingTooLong); - + // Expires if (params->putProperties && (params->putProperties->expires >= 0)) { time_t t = (time_t) params->putProperties->expires; @@ -523,16 +578,16 @@ static S3Status compose_standard_headers(const RequestParams *params, else { values->ifUnmodifiedSinceHeader[0] = 0; } - + // If-Match header do_get_header("If-Match: %s", ifMatchETag, ifMatchHeader, S3StatusBadIfMatchETag, S3StatusIfMatchETagTooLong); - + // If-None-Match header do_get_header("If-None-Match: %s", ifNotMatchETag, ifNoneMatchHeader, S3StatusBadIfNotMatchETag, S3StatusIfNotMatchETagTooLong); - + // Range header if (params->startByte || params->byteCount) { if (params->byteCount) { @@ -613,27 +668,41 @@ static void header_gnome_sort(const char **headers, int size) } -// Canonicalizes the x-amz- headers into the canonicalizedAmzHeaders buffer -static void canonicalize_amz_headers(RequestComputedValues *values) +// Canonicalizes the signature headers into the canonicalizedSignatureHeaders buffer +static void canonicalize_signature_headers(RequestComputedValues *values) { // Make a copy of the headers that will be sorted - const char *sortedHeaders[S3_MAX_METADATA_COUNT]; + const char *sortedHeaders[S3_MAX_METADATA_COUNT + 3]; memcpy(sortedHeaders, values->amzHeaders, (values->amzHeadersCount * sizeof(sortedHeaders[0]))); + // add the content-type header and host header + int headerCount = values->amzHeadersCount; + if (values->contentTypeHeader[0]) { + sortedHeaders[headerCount++] = values->contentTypeHeader; + } + if (values->hostHeader[0]) { + sortedHeaders[headerCount++] = values->hostHeader; + } + if (values->rangeHeader[0]) { + sortedHeaders[headerCount++] = values->rangeHeader; + } + // Now sort these - header_gnome_sort(sortedHeaders, values->amzHeadersCount); + header_gnome_sort(sortedHeaders, headerCount); // Now copy this sorted list into the buffer, all the while: // - folding repeated headers into single lines, and // - folding multiple lines // - removing the space after the colon - int lastHeaderLen = 0, i; - char *buffer = values->canonicalizedAmzHeaders; - for (i = 0; i < values->amzHeadersCount; i++) { + int lastHeaderLen = 0; + char *buffer = values->canonicalizedSignatureHeaders; + char *hbuf = values->signedHeaders; + for (int i = 0; i < headerCount; i++) { const char *header = sortedHeaders[i]; const char *c = header; + char v; // If the header names are the same, append the next value if ((i > 0) && !strncmp(header, sortedHeaders[i - 1], lastHeaderLen)) { @@ -646,8 +715,12 @@ static void canonicalize_amz_headers(RequestComputedValues *values) else { // Copy in everything up to the space in the ": " while (*c != ' ') { - *buffer++ = *c++; + v = tolower(*c++); + *buffer++ = v; + *hbuf++ = v; } + // replace the ":" with a ";" + *(hbuf - 1) = ';'; // Save the header len since it's a new header lastHeaderLen = c - header; // Skip the space @@ -662,7 +735,7 @@ static void canonicalize_amz_headers(RequestComputedValues *values) while (is_blank(*c)) { c++; } - // Also, what has most recently been copied into buffer amy + // Also, what has most recently been copied into buffer may // have been whitespace, and since we're folding whitespace // out around this newline sequence, back buffer up over // any whitespace it contains @@ -676,6 +749,9 @@ static void canonicalize_amz_headers(RequestComputedValues *values) // Finally, add the newline *buffer++ = '\n'; } + // Remove the extra trailing semicolon from the header name list + // and terminate the string. + *(hbuf - 1) = '\0'; // Terminate the buffer *buffer = 0; @@ -683,8 +759,7 @@ static void canonicalize_amz_headers(RequestComputedValues *values) // Canonicalizes the resource into params->canonicalizedResource -static void canonicalize_resource(const char *bucketName, - const char *subResource, +static void canonicalize_resource(const S3BucketContext *context, const char *urlEncodedKey, char *buffer) { @@ -694,9 +769,11 @@ static void canonicalize_resource(const char *bucketName, #define append(str) len += sprintf(&(buffer[len]), "%s", str) - if (bucketName && bucketName[0]) { - buffer[len++] = '/'; - append(bucketName); + if (context->uriStyle == S3UriStylePath) { + if (context->bucketName && context->bucketName[0]) { + buffer[len++] = '/'; + append(context->bucketName); + } } append("/"); @@ -705,12 +782,48 @@ static void canonicalize_resource(const char *bucketName, append(urlEncodedKey); } - if (subResource && subResource[0]) { - append("?"); - append(subResource); - } +#undef append } +// Canonicalize the query string part of the request into a buffer +static void canonicalize_query_string(const RequestParams *params, char *buffer) +{ + int len = 0; + + *buffer = 0; + +#define append(str) len += sprintf(&(buffer[len]), "%s", str) + + if (params->queryParams && params->queryParams[0]) { + char appendage[4]; + int foundEquals = 0; + for (unsigned long i = 0; i < strlen(params->queryParams); i++) { + char c = params->queryParams[i]; + if (isalnum(c) || (c == '_') || (c == '-') || (c == '~') + || (c == '.')) { + appendage[0] = c; + appendage[1] = '\0'; + } + else if ((c == '=') && !foundEquals) { + appendage[0] = c; + appendage[1] = '\0'; + foundEquals = 1; + } + else { + snprintf(appendage, 4, "%%%02X", c); + } + append(appendage); + } + } + + if (params->subResource && params->subResource[0]) { + if (params->queryParams && params->queryParams[0]) { + append("?"); + } + } + +#undef append +} // Convert an HttpRequestType to an HTTP Verb string static const char *http_request_type_to_verb(HttpRequestType requestType) @@ -733,57 +846,112 @@ static const char *http_request_type_to_verb(HttpRequestType requestType) // Composes the Authorization header for the request static S3Status compose_auth_header(const RequestParams *params, + const char *dateStr, RequestComputedValues *values) { - // We allow for: - // 17 bytes for HTTP-Verb + \n - // 129 bytes for Content-MD5 + \n - // 129 bytes for Content-Type + \n - // 1 byte for empty Date + \n - // CanonicalizedAmzHeaders & CanonicalizedResource - char signbuf[17 + 129 + 129 + 1 + - (sizeof(values->canonicalizedAmzHeaders) - 1) + - (sizeof(values->canonicalizedResource) - 1) + 1]; - int len = 0; - -#define signbuf_append(format, ...) \ - len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \ - format, __VA_ARGS__) + const char *httpMethod = http_request_type_to_verb(params->httpRequestType); + int canonicalRequestLen = strlen(httpMethod) + 1 + + strlen(values->canonicalURI) + 1 + + strlen(values->canonicalQueryString) + 1 + + strlen(values->canonicalizedSignatureHeaders) + 1 + + strlen(values->signedHeaders) + 1 + + 2 * CC_SHA256_DIGEST_LENGTH + 1; // 2 hex digits for each byte - signbuf_append - ("%s\n", http_request_type_to_verb(params->httpRequestType)); - - // For MD5 and Content-Type, use the value in the actual header, because - // it's already been trimmed - signbuf_append("%s\n", values->md5Header[0] ? - &(values->md5Header[sizeof("Content-MD5: ") - 1]) : ""); - - signbuf_append - ("%s\n", values->contentTypeHeader[0] ? - &(values->contentTypeHeader[sizeof("Content-Type: ") - 1]) : ""); - - signbuf_append("%s", "\n"); // Date - we always use x-amz-date + int len = 0; - signbuf_append("%s", values->canonicalizedAmzHeaders); + char canonicalRequest[canonicalRequestLen]; - signbuf_append("%s", values->canonicalizedResource); +#define buf_append(buf, format, ...) \ + len += snprintf(&(buf[len]), sizeof(buf) - len, \ + format, __VA_ARGS__) - // Generate an HMAC-SHA-1 of the signbuf - unsigned char hmac[20]; + canonicalRequest[0] = '\0'; + buf_append(canonicalRequest, "%s\n", httpMethod); + buf_append(canonicalRequest, "%s\n", values->canonicalURI); + buf_append(canonicalRequest, "%s\n", values->canonicalQueryString); + buf_append(canonicalRequest, "%s\n", values->canonicalizedSignatureHeaders); + buf_append(canonicalRequest, "%s\n", values->signedHeaders); + + buf_append(canonicalRequest, "%s", values->payloadHash); + +#ifdef SIGNATURE_DEBUG + printf("--\nCanonical Request:\n%s\n", canonicalRequest); +#endif + + len = 0; + unsigned char canonicalRequestHash[CC_SHA256_DIGEST_LENGTH]; + CC_SHA256(canonicalRequest, strlen(canonicalRequest), canonicalRequestHash); + char canonicalRequestHashHex[2 * CC_SHA256_DIGEST_LENGTH + 1]; + canonicalRequestHashHex[0] = '\0'; + for (int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + buf_append(canonicalRequestHashHex, "%02x", canonicalRequestHash[i]); + } + + // FIXME: determine correct region + const char *awsRegion = "us-east-1"; + char scope[SIGNATURE_SCOPE_SIZE + 1]; + snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request", dateStr, + awsRegion); + + char stringToSign[17 + 17 + SIGNATURE_SCOPE_SIZE + 1 + + strlen(canonicalRequestHashHex)]; + snprintf(stringToSign, sizeof(stringToSign), "AWS4-HMAC-SHA256\n%s\n%s\n%s", + dateStr, scope, canonicalRequestHashHex); + +#ifdef SIGNATURE_DEBUG + printf("--\nString to Sign:\n%s\n", stringToSign); +#endif + + const char *secretAccessKey = params->bucketContext.secretAccessKey; + char accessKey[strlen(secretAccessKey) + 5]; + snprintf(accessKey, sizeof(accessKey), "AWS4%s", secretAccessKey); + + unsigned char dateKey[CC_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, accessKey, strlen(accessKey), dateStr, 8, dateKey); + unsigned char dateRegionKey[CC_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateKey, CC_SHA256_DIGEST_LENGTH, awsRegion, + strlen(awsRegion), dateRegionKey); + unsigned char dateRegionServiceKey[CC_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateRegionKey, CC_SHA256_DIGEST_LENGTH, "s3", 2, + dateRegionServiceKey); + unsigned char signingKey[CC_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateRegionServiceKey, CC_SHA256_DIGEST_LENGTH, + "aws4_request", strlen("aws4_request"), signingKey); + + unsigned char finalSignature[CC_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, signingKey, CC_SHA256_DIGEST_LENGTH, stringToSign, + strlen(stringToSign), finalSignature); + + len = 0; + char finalSignatureHex[2 * CC_SHA256_DIGEST_LENGTH + 1]; + finalSignatureHex[0] = '\0'; + for (int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + buf_append(finalSignatureHex, "%02x", finalSignature[i]); + } + + char credential[strlen(params->bucketContext.accessKeyId) + 1 + 8 + 1 + + strlen(awsRegion) + 1 + 2 + 1 + 12 + 1]; + snprintf(credential, sizeof(credential), "%s/%.8s/%s/s3/aws4_request", + params->bucketContext.accessKeyId, dateStr, awsRegion); + + snprintf( + values->authorizationHeader, + sizeof(values->authorizationHeader), + "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + credential, values->signedHeaders, finalSignatureHex); + +#ifdef SIGNATURE_DEBUG + printf("--\nAuthorization Header:\n%s\n", values->authorizationHeader); + printf("--\nAMZ Headers:\n"); + for (int i = 0; i < values->amzHeadersCount; i++) { + printf("%s\n", values->amzHeaders[i]); + } +#endif - HMAC_SHA1(hmac, (unsigned char *) params->bucketContext.secretAccessKey, - strlen(params->bucketContext.secretAccessKey), - (unsigned char *) signbuf, len); + return S3StatusOK; - // Now base-64 encode the results - char b64[((20 + 1) * 4) / 3]; - int b64Len = base64Encode(hmac, 20, b64); - - snprintf(values->authorizationHeader, sizeof(values->authorizationHeader), - "Authorization: AWS %s:%.*s", params->bucketContext.accessKeyId, - b64Len, b64); +#undef buf_append - return S3StatusOK; } @@ -794,7 +962,7 @@ static S3Status compose_uri(char *buffer, int bufferSize, const char *subResource, const char *queryParams) { int len = 0; - + #define uri_append(fmt, ...) \ do { \ len += snprintf(&(buffer[len]), bufferSize - len, fmt, __VA_ARGS__); \ @@ -803,13 +971,13 @@ static S3Status compose_uri(char *buffer, int bufferSize, } \ } while (0) - uri_append("http%s://", + uri_append("http%s://", (bucketContext->protocol == S3ProtocolHTTP) ? "" : "s"); - const char *hostName = + const char *hostName = bucketContext->hostName ? bucketContext->hostName : defaultHostNameG; - if (bucketContext->bucketName && + if (bucketContext->bucketName && bucketContext->bucketName[0]) { if (bucketContext->uriStyle == S3UriStyleVirtualHost) { if (strchr(bucketContext->bucketName, '.') == NULL) { @@ -833,16 +1001,16 @@ static S3Status compose_uri(char *buffer, int bufferSize, uri_append("%s", "/"); uri_append("%s", urlEncodedKey); - + if (subResource && subResource[0]) { uri_append("?%s", subResource); } - + if (queryParams) { uri_append("%s%s", (subResource && subResource[0]) ? "&" : "?", queryParams); } - + return S3StatusOK; } @@ -1012,12 +1180,12 @@ static void request_deinitialize(Request *request) } -static S3Status request_get(const RequestParams *params, +static S3Status request_get(const RequestParams *params, const RequestComputedValues *values, Request **reqReturn) { Request *request = 0; - + // Try to get one from the request stack. We hold the lock for the // shortest time possible here. pthread_mutex_lock(&requestStackMutexG); @@ -1025,7 +1193,7 @@ static S3Status request_get(const RequestParams *params, if (requestStackCountG) { request = requestStackG[--requestStackCountG]; } - + pthread_mutex_unlock(&requestStackMutexG); // If we got one, deinitialize it for re-use @@ -1052,7 +1220,7 @@ static S3Status request_get(const RequestParams *params, request->status = S3StatusOK; S3Status status; - + // Start out with no headers request->headers = 0; @@ -1088,11 +1256,11 @@ static S3Status request_get(const RequestParams *params, response_headers_handler_initialize(&(request->responseHeadersHandler)); request->propertiesCallbackMade = 0; - + error_parser_initialize(&(request->errorParser)); *reqReturn = request; - + return S3StatusOK; } @@ -1201,14 +1369,24 @@ void request_perform(const RequestParams *params, S3RequestContext *context) return_status(status); } + char date[64]; + if (1) { + time_t now = time(NULL); + struct tm gmt; + gmtime_r(&now, &gmt); + strftime(date, sizeof(date), "%Y%m%dT%H%M%SZ", &gmt); + } + else { + snprintf(date, sizeof(date), "20130524T000000Z"); + } + // Compose the amz headers - if ((status = compose_amz_headers(params, &computed)) != S3StatusOK) { + if ((status = compose_amz_headers(params, date, &computed)) != S3StatusOK) { return_status(status); } // Compose standard headers - if ((status = compose_standard_headers - (params, &computed)) != S3StatusOK) { + if ((status = compose_standard_headers(params, &computed)) != S3StatusOK) { return_status(status); } @@ -1218,18 +1396,18 @@ void request_perform(const RequestParams *params, S3RequestContext *context) } // Compute the canonicalized amz headers - canonicalize_amz_headers(&computed); + canonicalize_signature_headers(&computed); // Compute the canonicalized resource - canonicalize_resource(params->bucketContext.bucketName, - params->subResource, computed.urlEncodedKey, - computed.canonicalizedResource); + canonicalize_resource(¶ms->bucketContext, computed.urlEncodedKey, + computed.canonicalURI); + canonicalize_query_string(params, computed.canonicalQueryString); // Compose Authorization header - if ((status = compose_auth_header(params, &computed)) != S3StatusOK) { + if ((status = compose_auth_header(params, date, &computed)) != S3StatusOK) { return_status(status); } - + // Get an initialized Request structure now if ((status = request_get(params, &computed, &request)) != S3StatusOK) { return_status(status); @@ -1239,12 +1417,12 @@ void request_perform(const RequestParams *params, S3RequestContext *context) } // Allow per-context override of verifyPeer if (verifyPeerRequest != verifyPeer) { - if ((curlstatus = curl_easy_setopt(request->curl, - CURLOPT_SSL_VERIFYPEER, - context->verifyPeer)) - != CURLE_OK) { - return_status(S3StatusFailedToInitializeRequest); - } + if ((curlstatus = curl_easy_setopt(request->curl, + CURLOPT_SSL_VERIFYPEER, + context->verifyPeer)) + != CURLE_OK) { + return_status(S3StatusFailedToInitializeRequest); + } } // If a RequestContext was provided, add the request to the curl multi @@ -1416,9 +1594,9 @@ S3Status S3_generate_authenticated_query_string // Compute canonicalized resource char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE]; - canonicalize_resource(bucketContext->bucketName, resource, urlEncodedKey, + canonicalize_resource(bucketContext, urlEncodedKey, canonicalizedResource); - + // We allow for: // 17 bytes for HTTP-Verb + \n // 1 byte for empty Content-MD5 + \n diff --git a/src/testsignature.c b/src/testsignature.c new file mode 100644 index 0000000..cbf90b7 --- /dev/null +++ b/src/testsignature.c @@ -0,0 +1,59 @@ +/* + * testsignature.c + * + * Created on: Sep 29, 2016 + * Author: eric + */ + +#include +#include +#include + +static S3Status responsePropertiesCallback( + const S3ResponseProperties *properties, void *callbackData) +{ + return S3StatusOK; +} + +static void responseCompleteCallback(S3Status status, + const S3ErrorDetails *error, + void *callbackData) +{ + if (error) { + printf("\nERROR\n"); + if (error->message) { + printf("%s\n", error->message); + } + if (error->extraDetailsCount > 0) { + for (int i = 0; i < error->extraDetailsCount; i++) { + printf("%s: %s\n", error->extraDetails[i].name, error->extraDetails[i].value); + } + } + } +} + +static S3Status getObjectDataCallback(int bufferSize, const char *buffer, + void *callbackData) +{ + return S3StatusOK; +} + +int main(int argc, char **argv) +{ + S3_initialize(NULL, S3_INIT_ALL, NULL); + + S3BucketContext bucketContext = + { 0, "examplebucket", S3ProtocolHTTPS, S3UriStyleVirtualHost, + "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 0 }; + + S3GetObjectHandler getObjectHandler = + { + { &responsePropertiesCallback, &responseCompleteCallback }, + &getObjectDataCallback + }; + + S3_get_object(&bucketContext, "test.txt", NULL, 0, 10, NULL, + &getObjectHandler, NULL); + + S3_deinitialize(); +} diff --git a/src/util.c b/src/util.c index 590d037..a9d2bc7 100644 --- a/src/util.c +++ b/src/util.c @@ -419,7 +419,7 @@ void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len, { unsigned char kopad[64], kipad[64]; int i; - + if (key_len > 64) { key_len = 64; } @@ -437,7 +437,7 @@ void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len, unsigned char digest[20]; SHA1Context context; - + SHA1_init(&context); SHA1_update(&context, kipad, 64); SHA1_update(&context, message, message_len); From 1b714e0af728d2481d731b95f335516a7a27c5a8 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 29 Sep 2016 15:35:02 -0600 Subject: [PATCH 05/57] adapted v4 signature construction for Linux --- inc/request.h | 4 +-- src/request.c | 92 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 70 insertions(+), 26 deletions(-) diff --git a/inc/request.h b/inc/request.h index b54e945..bbe56a4 100644 --- a/inc/request.h +++ b/inc/request.h @@ -74,10 +74,10 @@ typedef struct RequestParams const S3GetConditions *getConditions; // Start byte - uint64_t startByte; + size_t startByte; // Byte count - uint64_t byteCount; + size_t byteCount; // Put properties const S3PutProperties *putProperties; diff --git a/src/request.c b/src/request.c index b9e63c5..daedfda 100644 --- a/src/request.c +++ b/src/request.c @@ -35,6 +35,11 @@ #ifdef __APPLE__ #include +#define S3_SHA256_DIGEST_LENGTH CC_SHA256_DIGEST_LENGTH +#else +#include +#include +#define S3_SHA256_DIGEST_LENGTH SHA256_DIGEST_LENGTH #endif #define USER_AGENT_SIZE 256 @@ -126,7 +131,7 @@ typedef struct RequestComputedValues char hostHeader[128]; // Hex string of hash of request payload - char payloadHash[CC_SHA256_DIGEST_LENGTH * 2 + 1]; + char payloadHash[S3_SHA256_DIGEST_LENGTH * 2 + 1]; } RequestComputedValues; @@ -285,14 +290,15 @@ static S3Status append_amz_header(RequestComputedValues *values, return S3StatusMetaDataHeadersTooLong; } - for (unsigned long i = 0; i < strlen(headerStr); i++) { + unsigned long i = 0; + for (; i < strlen(headerStr); i++) { values->amzHeadersRaw[rawPos++] = tolower(headerStr[i]); } strcat(&(values->amzHeadersRaw[rawPos]), ": "); rawPos += 2; - for (unsigned long i = 0; i < strlen(headerValue); i++) { + for (i = 0; i < strlen(headerValue); i++) { values->amzHeadersRaw[rawPos++] = headerValue[i]; } rawPos--; @@ -375,7 +381,7 @@ static S3Status compose_amz_headers(const RequestParams *params, // If byteCount != 0 then we're just copying a range, add header if (params->byteCount > 0) { char byteRange[S3_MAX_METADATA_SIZE]; - snprintf(byteRange, sizeof(byteRange), "bytes=%lld-%lld", + snprintf(byteRange, sizeof(byteRange), "bytes=%zd-%zd", params->startByte, params->startByte + params->byteCount); append_amz_header(values, 0, "x-amz-copy-source-range", byteRange); } @@ -396,10 +402,15 @@ static S3Status compose_amz_headers(const RequestParams *params, || params->httpRequestType == HttpRequestTypeDELETE || params->httpRequestType == HttpRequestTypeHEAD) { // empty payload - unsigned char md[CC_SHA256_DIGEST_LENGTH]; + unsigned char md[S3_SHA256_DIGEST_LENGTH]; +#ifdef __APPLE__ CC_SHA256("", 0, md); +#else + SHA256((const unsigned char*) "", 0, md); +#endif values->payloadHash[0] = '\0'; - for (int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + int i = 0; + for (; i < S3_SHA256_DIGEST_LENGTH; i++) { snprintf(&(values->payloadHash[i * 2]), 3, "%02x", md[i]); } } @@ -699,7 +710,8 @@ static void canonicalize_signature_headers(RequestComputedValues *values) int lastHeaderLen = 0; char *buffer = values->canonicalizedSignatureHeaders; char *hbuf = values->signedHeaders; - for (int i = 0; i < headerCount; i++) { + int i = 0; + for (; i < headerCount; i++) { const char *header = sortedHeaders[i]; const char *c = header; char v; @@ -797,7 +809,8 @@ static void canonicalize_query_string(const RequestParams *params, char *buffer) if (params->queryParams && params->queryParams[0]) { char appendage[4]; int foundEquals = 0; - for (unsigned long i = 0; i < strlen(params->queryParams); i++) { + unsigned long i = 0; + for (; i < strlen(params->queryParams); i++) { char c = params->queryParams[i]; if (isalnum(c) || (c == '_') || (c == '-') || (c == '~') || (c == '.')) { @@ -855,7 +868,7 @@ static S3Status compose_auth_header(const RequestParams *params, strlen(values->canonicalQueryString) + 1 + strlen(values->canonicalizedSignatureHeaders) + 1 + strlen(values->signedHeaders) + 1 + - 2 * CC_SHA256_DIGEST_LENGTH + 1; // 2 hex digits for each byte + 2 * S3_SHA256_DIGEST_LENGTH + 1; // 2 hex digits for each byte int len = 0; @@ -879,11 +892,17 @@ static S3Status compose_auth_header(const RequestParams *params, #endif len = 0; - unsigned char canonicalRequestHash[CC_SHA256_DIGEST_LENGTH]; + unsigned char canonicalRequestHash[S3_SHA256_DIGEST_LENGTH]; +#ifdef __APPLE__ CC_SHA256(canonicalRequest, strlen(canonicalRequest), canonicalRequestHash); - char canonicalRequestHashHex[2 * CC_SHA256_DIGEST_LENGTH + 1]; +#else + const unsigned char *rqstData = (const unsigned char*) canonicalRequest; + SHA256(rqstData, strlen(canonicalRequest), canonicalRequestHash); +#endif + char canonicalRequestHashHex[2 * S3_SHA256_DIGEST_LENGTH + 1]; canonicalRequestHashHex[0] = '\0'; - for (int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + int i = 0; + for (; i < S3_SHA256_DIGEST_LENGTH; i++) { buf_append(canonicalRequestHashHex, "%02x", canonicalRequestHash[i]); } @@ -906,26 +925,51 @@ static S3Status compose_auth_header(const RequestParams *params, char accessKey[strlen(secretAccessKey) + 5]; snprintf(accessKey, sizeof(accessKey), "AWS4%s", secretAccessKey); - unsigned char dateKey[CC_SHA256_DIGEST_LENGTH]; +#ifdef __APPLE__ + unsigned char dateKey[S3_SHA256_DIGEST_LENGTH]; CCHmac(kCCHmacAlgSHA256, accessKey, strlen(accessKey), dateStr, 8, dateKey); - unsigned char dateRegionKey[CC_SHA256_DIGEST_LENGTH]; - CCHmac(kCCHmacAlgSHA256, dateKey, CC_SHA256_DIGEST_LENGTH, awsRegion, + unsigned char dateRegionKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateKey, S3_SHA256_DIGEST_LENGTH, awsRegion, strlen(awsRegion), dateRegionKey); - unsigned char dateRegionServiceKey[CC_SHA256_DIGEST_LENGTH]; - CCHmac(kCCHmacAlgSHA256, dateRegionKey, CC_SHA256_DIGEST_LENGTH, "s3", 2, + unsigned char dateRegionServiceKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateRegionKey, S3_SHA256_DIGEST_LENGTH, "s3", 2, dateRegionServiceKey); - unsigned char signingKey[CC_SHA256_DIGEST_LENGTH]; - CCHmac(kCCHmacAlgSHA256, dateRegionServiceKey, CC_SHA256_DIGEST_LENGTH, + unsigned char signingKey[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, dateRegionServiceKey, S3_SHA256_DIGEST_LENGTH, "aws4_request", strlen("aws4_request"), signingKey); - unsigned char finalSignature[CC_SHA256_DIGEST_LENGTH]; - CCHmac(kCCHmacAlgSHA256, signingKey, CC_SHA256_DIGEST_LENGTH, stringToSign, - strlen(stringToSign), finalSignature); + unsigned char finalSignature[S3_SHA256_DIGEST_LENGTH]; + CCHmac(kCCHmacAlgSHA256, signingKey, S3_SHA256_DIGEST_LENGTH, stringToSign, + strlen(stringToSign), finalSignature); +#else + const EVP_MD *sha256evp = EVP_sha256(); + unsigned char dateKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, accessKey, strlen(accessKey), + (const unsigned char*) dateStr, 8, dateKey, + NULL); + unsigned char dateRegionKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, dateKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) awsRegion, strlen(awsRegion), dateRegionKey, + NULL); + unsigned char dateRegionServiceKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, dateRegionKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) "s3", 2, dateRegionServiceKey, NULL); + unsigned char signingKey[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, dateRegionServiceKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) "aws4_request", strlen("aws4_request"), + signingKey, + NULL); + + unsigned char finalSignature[S3_SHA256_DIGEST_LENGTH]; + HMAC(sha256evp, signingKey, S3_SHA256_DIGEST_LENGTH, + (const unsigned char*) stringToSign, strlen(stringToSign), + finalSignature, NULL); +#endif len = 0; - char finalSignatureHex[2 * CC_SHA256_DIGEST_LENGTH + 1]; + char finalSignatureHex[2 * S3_SHA256_DIGEST_LENGTH + 1]; finalSignatureHex[0] = '\0'; - for (int i = 0; i < CC_SHA256_DIGEST_LENGTH; i++) { + for (i = 0; i < S3_SHA256_DIGEST_LENGTH; i++) { buf_append(finalSignatureHex, "%02x", finalSignature[i]); } From 34d11b2e0f1440faa6230dc0d8312439022e5efb Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 29 Sep 2016 19:10:26 -0600 Subject: [PATCH 06/57] add auth. region to relevant API calls --- GNUmakefile.osx | 15 ++++++++++++- inc/libs3.h | 41 ++++++++++++++++++++++++++---------- src/acl.c | 8 ++++--- src/bucket.c | 32 ++++++++++++++++------------ src/multipart.c | 28 ++++++++++++++---------- src/object.c | 15 ++++++++----- src/request.c | 8 ++++--- src/s3.c | 27 +++++++++++++++++++----- src/service.c | 8 ++++--- src/service_access_logging.c | 6 ++++-- src/testsignature.c | 15 +++++++------ 11 files changed, 139 insertions(+), 64 deletions(-) diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 76a45e9..ab1aff2 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -102,6 +102,7 @@ endif ifndef LIBXML2_LIBS LIBXML2_LIBS := $(shell xml2-config --libs) + LIBXML2_LIBS := $(filter-out -L$(shell xcrun --show-sdk-path)/usr/lib, $(LIBXML2_LIBS)) endif ifndef LIBXML2_CFLAGS @@ -194,6 +195,14 @@ uninstall: # -------------------------------------------------------------------------- # Compile target patterns +$(BUILD)/obj/test%.o: src/test%.c + $(QUIET_ECHO) $@: Compiling test object + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ gcc $(CFLAGS) -Wno-error=unused-parameter -M -MG -MQ $@ \ + -DCOMPILINGDEPENDENCIES -o $(BUILD)/dep/$(<:%.c=%.d) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) gcc $(CFLAGS) -Wno-error=unused-parameter -o $@ -c $< + $(BUILD)/obj/%.o: src/%.c $(QUIET_ECHO) $@: Compiling object @ mkdir -p $(dir $(BUILD)/dep/$<) @@ -267,13 +276,17 @@ $(BUILD)/include/libs3.h: inc/libs3.h # Test targets .PHONY: test -test: $(BUILD)/bin/testsimplexml +test: $(BUILD)/bin/testsimplexml $(BUILD)/bin/testsignature $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) $(QUIET_ECHO) $@: Building executable @ mkdir -p $(dir $@) $(VERBOSE_SHOW) gcc -o $@ $^ $(LIBXML2_LIBS) +$(BUILD)/bin/testsignature: $(BUILD)/obj/testsignature.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) gcc -o $@ $^ $(LDFLAGS) # -------------------------------------------------------------------------- # Clean target diff --git a/inc/libs3.h b/inc/libs3.h index 4e0aaad..69066d5 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -228,6 +228,12 @@ extern "C" { #define S3_INIT_ALL (S3_INIT_WINSOCK) +/** + * The default region identifier used to scope the signing key + */ +#define S3_DEFAULT_REGION "us-east-1" + + /** ************************************************************************** * Enumerations ************************************************************************** **/ @@ -711,6 +717,12 @@ typedef struct S3BucketContext * The Amazon Security Token used to generate Temporary Security Credentials **/ const char *securityToken; + + /** + * The AWS region to which to scope the signing key used for authorization. + * If NULL, the default region ("us-east-1") will be used. + */ + const char *authRegion; } S3BucketContext; @@ -1740,6 +1752,7 @@ S3Status S3_generate_authenticated_query_string * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. + * @param authRegion is the AWS region to use for the authorization signature * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. @@ -1750,10 +1763,10 @@ S3Status S3_generate_authenticated_query_string **/ void S3_list_service(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, - const char *hostName, S3RequestContext *requestContext, - const S3ListServiceHandler *handler, - void *callbackData); - + const char *hostName, const char *authRegion, + S3RequestContext *requestContext, + const S3ListServiceHandler *handler, void *callbackData); + /** ************************************************************************** * Bucket Functions @@ -1773,6 +1786,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. * @param bucketName is the bucket name to test + * @param authRegion is the AWS region to use for the authorization signature * @param locationConstraintReturnSize gives the number of bytes in the * locationConstraintReturn parameter * @param locationConstraintReturn provides the location into which to write @@ -1792,13 +1806,14 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, **/ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, - const char *securityToken, const char *hostName, - const char *bucketName, int locationConstraintReturnSize, + const char *securityToken, const char *hostName, + const char *bucketName, const char *authRegion, + int locationConstraintReturnSize, char *locationConstraintReturn, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); - + /** * Creates a new bucket. * @@ -1812,6 +1827,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. * @param bucketName is the name of the bucket to be created + * @param authRegion is the AWS region to use for the authorization signature * @param cannedAcl gives the "REST canned ACL" to use for the created bucket * @param locationConstraint if non-NULL, gives the geographic location for * the bucket to create. @@ -1826,7 +1842,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *bucketName, - S3CannedAcl cannedAcl, const char *locationConstraint, + const char *authRegion, S3CannedAcl cannedAcl, + const char *locationConstraint, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); @@ -1846,6 +1863,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. * @param bucketName is the name of the bucket to be deleted + * @param authRegion is the AWS region to use for the authorization signature * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. @@ -1856,8 +1874,9 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, **/ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, - const char *securityToken, const char *hostName, - const char *bucketName, S3RequestContext *requestContext, + const char *securityToken, const char *hostName, + const char *bucketName, const char *authRegion, + S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); @@ -1917,7 +1936,7 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, const S3PutProperties *putProperties, S3RequestContext *requestContext, const S3PutObjectHandler *handler, void *callbackData); - + /** * Copies an object from one location to another. The object may be copied diff --git a/src/acl.c b/src/acl.c index b27b8db..c2eeb08 100644 --- a/src/acl.c +++ b/src/acl.c @@ -128,7 +128,8 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams "acl", // subResource @@ -299,7 +300,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); return; } - + // Convert aclGrants to XML document S3Status status = generateAclXmlDocument (ownerId, ownerDisplayName, aclGrantCount, aclGrants, @@ -327,7 +328,8 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams "acl", // subResource diff --git a/src/bucket.c b/src/bucket.c index dcbcf2f..66c455d 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -107,7 +107,8 @@ static void testBucketCompleteCallback(S3Status requestStatus, void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, - const char *bucketName, int locationConstraintReturnSize, + const char *bucketName, const char *authRegion, + int locationConstraintReturnSize, char *locationConstraintReturn, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData) @@ -140,7 +141,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, uriStyle, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams "location", // subResource @@ -223,11 +225,11 @@ static void createBucketCompleteCallback(S3Status requestStatus, free(cbData); } - void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *bucketName, - S3CannedAcl cannedAcl, const char *locationConstraint, + const char *authRegion, S3CannedAcl cannedAcl, + const char *locationConstraint, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData) { @@ -254,7 +256,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, else { cbData->docLen = 0; } - + // Set up S3PutProperties S3PutProperties properties = { @@ -269,7 +271,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, 0, // metaData 0 // useServerSideEncryption }; - + // Set up the RequestParams RequestParams params = { @@ -280,7 +282,8 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, S3UriStylePath, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams 0, // subResource @@ -317,7 +320,7 @@ static S3Status deleteBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { DeleteBucketData *dbData = (DeleteBucketData *) callbackData; - + return (*(dbData->responsePropertiesCallback)) (responseProperties, dbData->callbackData); } @@ -338,12 +341,13 @@ static void deleteBucketCompleteCallback(S3Status requestStatus, void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *accessKeyId, const char *secretAccessKey, - const char *securityToken, const char *hostName, - const char *bucketName, S3RequestContext *requestContext, + const char *securityToken, const char *hostName, + const char *bucketName, const char *authRegion, + S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - DeleteBucketData *dbData = + DeleteBucketData *dbData = (DeleteBucketData *) malloc(sizeof(DeleteBucketData)); if (!dbData) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -364,7 +368,8 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, uriStyle, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams 0, // subResource @@ -724,7 +729,8 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key queryParams[0] ? queryParams : 0, // queryParams 0, // subResource diff --git a/src/multipart.c b/src/multipart.c index b21e0f5..df88a7e 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -112,12 +112,13 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, { HttpRequestTypePOST, // httpRequestType { bucketContext->hostName, // hostName - bucketContext->bucketName, // bucketName + bucketContext->bucketName, // bucketName bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams "uploads", // subResource @@ -155,8 +156,9 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams subResource, // subResource @@ -200,8 +202,9 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams subResource, // subResource @@ -337,8 +340,9 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId - bucketContext->secretAccessKey, //secretAccessKey - bucketContext->securityToken }, // secretToken + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams subResource, // subResource @@ -930,7 +934,8 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key queryParams[0] ? queryParams : 0, // queryParams "uploads", // subResource @@ -1051,7 +1056,8 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key queryParams[0] ? queryParams : 0, // queryParams subResource, // subResource diff --git a/src/object.c b/src/object.c index b1e640b..5e167fc 100644 --- a/src/object.c +++ b/src/object.c @@ -48,7 +48,8 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -237,7 +238,8 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion destinationKey ? destinationKey : key, // key 0, // queryParams subRsrc, // subResource @@ -278,7 +280,8 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -317,7 +320,8 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource @@ -356,7 +360,8 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion key, // key 0, // queryParams 0, // subResource diff --git a/src/request.c b/src/request.c index daedfda..a5f5793 100644 --- a/src/request.c +++ b/src/request.c @@ -906,8 +906,10 @@ static S3Status compose_auth_header(const RequestParams *params, buf_append(canonicalRequestHashHex, "%02x", canonicalRequestHash[i]); } - // FIXME: determine correct region - const char *awsRegion = "us-east-1"; + const char *awsRegion = S3_DEFAULT_REGION; + if (params->bucketContext.authRegion) { + awsRegion = params->bucketContext.authRegion; + } char scope[SIGNATURE_SCOPE_SIZE + 1]; snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request", dateStr, awsRegion); @@ -1377,7 +1379,7 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, snprintf(userAgentG, sizeof(userAgentG), "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)", userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform); - + return S3StatusOK; } diff --git a/src/s3.c b/src/s3.c index 0493b31..7d108af 100644 --- a/src/s3.c +++ b/src/s3.c @@ -913,7 +913,7 @@ static void list_service(int allDetails) }; do { - S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, 0, + S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, 0, 0, &listServiceHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -957,7 +957,7 @@ static void test_bucket(int argc, char **argv, int optindex) char locationConstraint[64]; do { S3_test_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, 0, - 0, bucketName, sizeof(locationConstraint), + 0, bucketName, 0, sizeof(locationConstraint), locationConstraint, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -1056,7 +1056,7 @@ static void create_bucket(int argc, char **argv, int optindex) do { S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG, 0, - 0, bucketName, cannedAcl, locationConstraint, 0, + 0, bucketName, 0, cannedAcl, locationConstraint, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -1096,7 +1096,7 @@ static void delete_bucket(int argc, char **argv, int optindex) do { S3_delete_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0, 0, bucketName, 0, &responseHandler, 0); + 0, 0, bucketName, 0, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -1253,6 +1253,7 @@ static void list_bucket(const char *bucketName, const char *prefix, uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -1679,6 +1680,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -1801,6 +1803,7 @@ static void list_parts(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -1900,6 +1903,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -1963,6 +1967,7 @@ static void delete_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -2089,6 +2094,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -2351,6 +2357,7 @@ static void put_object(int argc, char **argv, int optindex, uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -2480,6 +2487,7 @@ static void put_object(int argc, char **argv, int optindex, uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -2614,6 +2622,7 @@ static void copy_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; S3ListBucketHandler listBucketHandler = @@ -2760,6 +2769,7 @@ static void copy_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -2942,6 +2952,7 @@ static void get_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3017,6 +3028,7 @@ static void head_object(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3097,6 +3109,7 @@ static void generate_query_string(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3200,6 +3213,7 @@ void get_acl(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3364,6 +3378,7 @@ void set_acl(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3457,6 +3472,7 @@ void get_logging(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3624,6 +3640,7 @@ void set_logging(int argc, char **argv, int optindex) uriStyleG, accessKeyIdG, secretAccessKeyG, + 0, 0 }; @@ -3701,7 +3718,7 @@ int main(int argc, char **argv) } const char *command = argv[optind++]; - + if (!strcmp(command, "help")) { fprintf(stdout, "\ns3 is a program for performing single requests " "to Amazon S3.\n"); diff --git a/src/service.c b/src/service.c index cc505d8..b4b3123 100644 --- a/src/service.c +++ b/src/service.c @@ -100,7 +100,7 @@ static S3Status propertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { XmlCallbackData *cbData = (XmlCallbackData *) callbackData; - + return (*(cbData->responsePropertiesCallback)) (responseProperties, cbData->callbackData); } @@ -132,7 +132,8 @@ static void completeCallback(S3Status requestStatus, void S3_list_service(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, - const char *hostName, S3RequestContext *requestContext, + const char *hostName, const char *authRegion, + S3RequestContext *requestContext, const S3ListServiceHandler *handler, void *callbackData) { // Create and set up the callback data @@ -167,7 +168,8 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, S3UriStylePath, // uriStyle accessKeyId, // accessKeyId secretAccessKey, // secretAccessKey - securityToken }, // securityToken + securityToken, // securityToken + authRegion }, // authRegion 0, // key 0, // queryParams 0, // subResource diff --git a/src/service_access_logging.c b/src/service_access_logging.c index 28d0079..dc90a03 100644 --- a/src/service_access_logging.c +++ b/src/service_access_logging.c @@ -329,7 +329,8 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key 0, // queryParams "logging", // subResource @@ -534,7 +535,8 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, bucketContext->uriStyle, // uriStyle bucketContext->accessKeyId, // accessKeyId bucketContext->secretAccessKey, // secretAccessKey - bucketContext->securityToken }, // securityToken + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion 0, // key 0, // queryParams "logging", // subResource diff --git a/src/testsignature.c b/src/testsignature.c index cbf90b7..b72a963 100644 --- a/src/testsignature.c +++ b/src/testsignature.c @@ -26,7 +26,8 @@ static void responseCompleteCallback(S3Status status, } if (error->extraDetailsCount > 0) { for (int i = 0; i < error->extraDetailsCount; i++) { - printf("%s: %s\n", error->extraDetails[i].name, error->extraDetails[i].value); + printf("%s: %s\n", error->extraDetails[i].name, + error->extraDetails[i].value); } } } @@ -43,17 +44,17 @@ int main(int argc, char **argv) S3_initialize(NULL, S3_INIT_ALL, NULL); S3BucketContext bucketContext = - { 0, "examplebucket", S3ProtocolHTTPS, S3UriStyleVirtualHost, - "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", 0 }; + { 0, "examplebucket", S3ProtocolHTTPS, S3UriStyleVirtualHost, + "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + 0, 0 }; S3GetObjectHandler getObjectHandler = { - { &responsePropertiesCallback, &responseCompleteCallback }, - &getObjectDataCallback - }; + { &responsePropertiesCallback, &responseCompleteCallback }, + &getObjectDataCallback }; S3_get_object(&bucketContext, "test.txt", NULL, 0, 10, NULL, - &getObjectHandler, NULL); + &getObjectHandler, NULL); S3_deinitialize(); } From e67a3f6de7cb80897b3f7d21a7d6edbea1c9a67e Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Thu, 29 Sep 2016 19:46:01 -0600 Subject: [PATCH 07/57] add openssl libs to makefile, fix test code on Linux --- GNUmakefile | 20 ++++++++++++++++++-- src/testsignature.c | 5 ++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 4d3e2e5..ae0734e 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -119,6 +119,9 @@ ifndef LIBXML2_CFLAGS LIBXML2_CFLAGS := $(shell xml2-config --cflags) endif +ifndef OPENSSL_LIBS + OPENSSL_LIBS := -lssl -lcrypto +endif # -------------------------------------------------------------------------- # These CFLAGS assume a GNU compiler. For other compilers, write a script @@ -142,7 +145,7 @@ CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ -D_ISOC99_SOURCE \ -D_POSIX_C_SOURCE=200112L -LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) -lpthread +LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread # -------------------------------------------------------------------------- @@ -204,6 +207,14 @@ uninstall: # -------------------------------------------------------------------------- # Compile target patterns +$(BUILD)/obj/test%.o: src/test%.c + $(QUIET_ECHO) $@: Compiling test object + @ mkdir -p $(dir $(BUILD)/dep/$<) + @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ + -o $(BUILD)/dep/$(<:%.c=%.d) -c $< + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -Wno-error=unused-parameter $(CFLAGS) -o $@ -c $< + $(BUILD)/obj/%.o: src/%.c $(QUIET_ECHO) $@: Compiling object @ mkdir -p $(dir $(BUILD)/dep/$<) @@ -275,13 +286,18 @@ $(BUILD)/include/libs3.h: inc/libs3.h # Test targets .PHONY: test -test: $(BUILD)/bin/testsimplexml +test: $(BUILD)/bin/testsimplexml $(BUILD)/bin/testsignature $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) $(QUIET_ECHO) $@: Building executable @ mkdir -p $(dir $@) $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS) +$(BUILD)/bin/testsignature: $(BUILD)/obj/testsignature.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) + # -------------------------------------------------------------------------- # Clean target diff --git a/src/testsignature.c b/src/testsignature.c index b72a963..f80e6da 100644 --- a/src/testsignature.c +++ b/src/testsignature.c @@ -25,7 +25,8 @@ static void responseCompleteCallback(S3Status status, printf("%s\n", error->message); } if (error->extraDetailsCount > 0) { - for (int i = 0; i < error->extraDetailsCount; i++) { + int i = 0; + for (; i < error->extraDetailsCount; i++) { printf("%s: %s\n", error->extraDetails[i].name, error->extraDetails[i].value); } @@ -57,4 +58,6 @@ int main(int argc, char **argv) &getObjectHandler, NULL); S3_deinitialize(); + + return 0; } From 0aedf1fe0e675f84b435899a77c7af3946856d54 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 30 Sep 2016 09:17:35 -0600 Subject: [PATCH 08/57] support the generate_query_string operation with the new authorization/signature logic --- inc/libs3.h | 5 +- inc/request.h | 3 +- inc/util.h | 7 +- src/request.c | 256 +++++++++++++++++++++++++++----------------------- src/s3.c | 209 ++++++++++++++++++++++------------------- 5 files changed, 261 insertions(+), 219 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index 69066d5..6946507 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -1725,6 +1725,8 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, * @param resource gives a sub-resource to be fetched for the request, or NULL * for none. This should be of the form "?", i.e. * "?torrent". + * @param httpMethod the HTTP request method that will be used with the + * generated query string (e.g. "GET"). * @return One of: * S3StatusUriTooLong if, due to an internal error, the generated URI * is longer than S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in @@ -1733,7 +1735,8 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, **/ S3Status S3_generate_authenticated_query_string (char *buffer, const S3BucketContext *bucketContext, - const char *key, int64_t expires, const char *resource); + const char *key, int64_t expires, const char *resource, + const char *httpMethod); /** ************************************************************************** diff --git a/inc/request.h b/inc/request.h index bbe56a4..7d9cb03 100644 --- a/inc/request.h +++ b/inc/request.h @@ -40,7 +40,8 @@ typedef enum HttpRequestTypePUT, HttpRequestTypeCOPY, HttpRequestTypeDELETE, - HttpRequestTypePOST + HttpRequestTypePOST, + HttpRequestTypeInvalid } HttpRequestType; diff --git a/inc/util.h b/inc/util.h index 854b9d9..f63eaf3 100644 --- a/inc/util.h +++ b/inc/util.h @@ -40,7 +40,6 @@ #define ACS_GROUP_LOG_DELIVERY ACS_URL "s3/LogDelivery" - // Derived from S3 documentation // This is the maximum number of bytes needed in a "compacted meta header" @@ -64,6 +63,12 @@ #define MAX_CANONICALIZED_RESOURCE_SIZE \ (1 + 255 + 1 + MAX_URLENCODED_KEY_SIZE + (sizeof("?torrent") - 1) + 1) +#define MAX_ACCESS_KEY_ID_LENGTH 32 + +// Maximum length of a credential string +// ///s3/aws4_request +#define MAX_CREDENTIAL_SIZE \ + (MAX_ACCESS_KEY_ID_LENGTH + 1) + 8 + 1 + 32 + sizeof("/s3/aws4_request") // Utilities ----------------------------------------------------------------- diff --git a/src/request.c b/src/request.c index a5f5793..1897c97 100644 --- a/src/request.c +++ b/src/request.c @@ -127,6 +127,15 @@ typedef struct RequestComputedValues // Authorization header char authorizationHeader[256]; + // Request date stamp + char requestDateISO8601[64]; + + // Credential used for authorization signature + char authCredential[MAX_CREDENTIAL_SIZE + 1]; + + // Computed request signature (hex string) + char requestSignatureHex[2 * S3_SHA256_DIGEST_LENGTH + 1]; + // Host header char hostHeader[128]; @@ -322,7 +331,7 @@ static S3Status append_amz_header(RequestComputedValues *values, // these headers in params->amzHeaders (and also sets params->amzHeadersCount // to be the count of the total number of x-amz- headers thus created). static S3Status compose_amz_headers(const RequestParams *params, - const char *dateStr, + int forceUnsignedPayload, RequestComputedValues *values) { const S3PutProperties *properties = params->putProperties; @@ -367,7 +376,7 @@ static S3Status compose_amz_headers(const RequestParams *params, } // Add the x-amz-date header - append_amz_header(values, 0, "x-amz-date", dateStr); + append_amz_header(values, 0, "x-amz-date", values->requestDateISO8601); if (params->httpRequestType == HttpRequestTypeCOPY) { // Add the x-amz-copy-source header @@ -397,10 +406,11 @@ static S3Status compose_amz_headers(const RequestParams *params, params->bucketContext.securityToken); } - if (params->httpRequestType == HttpRequestTypeGET - || params->httpRequestType == HttpRequestTypeCOPY - || params->httpRequestType == HttpRequestTypeDELETE - || params->httpRequestType == HttpRequestTypeHEAD) { + if (!forceUnsignedPayload + && (params->httpRequestType == HttpRequestTypeGET + || params->httpRequestType == HttpRequestTypeCOPY + || params->httpRequestType == HttpRequestTypeDELETE + || params->httpRequestType == HttpRequestTypeHEAD)) { // empty payload unsigned char md[S3_SHA256_DIGEST_LENGTH]; #ifdef __APPLE__ @@ -798,7 +808,8 @@ static void canonicalize_resource(const S3BucketContext *context, } // Canonicalize the query string part of the request into a buffer -static void canonicalize_query_string(const RequestParams *params, char *buffer) +static void canonicalize_query_string(const char *queryParams, + const char *subResource, char *buffer) { int len = 0; @@ -806,12 +817,12 @@ static void canonicalize_query_string(const RequestParams *params, char *buffer) #define append(str) len += sprintf(&(buffer[len]), "%s", str) - if (params->queryParams && params->queryParams[0]) { + if (queryParams && queryParams[0]) { char appendage[4]; int foundEquals = 0; unsigned long i = 0; - for (; i < strlen(params->queryParams); i++) { - char c = params->queryParams[i]; + for (; i < strlen(queryParams); i++) { + char c = queryParams[i]; if (isalnum(c) || (c == '_') || (c == '-') || (c == '~') || (c == '.')) { appendage[0] = c; @@ -829,15 +840,44 @@ static void canonicalize_query_string(const RequestParams *params, char *buffer) } } - if (params->subResource && params->subResource[0]) { - if (params->queryParams && params->queryParams[0]) { + if (subResource && subResource[0]) { + if (queryParams && queryParams[0]) { append("?"); } + append(subResource); } #undef append } + +static HttpRequestType http_request_method_to_type(const char *method) +{ + if (!method) { + return HttpRequestTypeInvalid; + } + if (strcmp(method, "POST") == 0) { + return HttpRequestTypePOST; + } + else if (strcmp(method, "GET") == 0) { + return HttpRequestTypeGET; + } + else if (strcmp(method, "HEAD") == 0) { + return HttpRequestTypeHEAD; + } + else if (strcmp(method, "PUT") == 0) { + return HttpRequestTypePUT; + } + else if (strcmp(method, "COPY") == 0) { + return HttpRequestTypeCOPY; + } + else if (strcmp(method, "DELETE") == 0) { + return HttpRequestTypeDELETE; + } + return HttpRequestTypeInvalid; +} + + // Convert an HttpRequestType to an HTTP Verb string static const char *http_request_type_to_verb(HttpRequestType requestType) { @@ -859,7 +899,6 @@ static const char *http_request_type_to_verb(HttpRequestType requestType) // Composes the Authorization header for the request static S3Status compose_auth_header(const RequestParams *params, - const char *dateStr, RequestComputedValues *values) { const char *httpMethod = http_request_type_to_verb(params->httpRequestType); @@ -911,13 +950,13 @@ static S3Status compose_auth_header(const RequestParams *params, awsRegion = params->bucketContext.authRegion; } char scope[SIGNATURE_SCOPE_SIZE + 1]; - snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request", dateStr, - awsRegion); + snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request", + values->requestDateISO8601, awsRegion); char stringToSign[17 + 17 + SIGNATURE_SCOPE_SIZE + 1 + strlen(canonicalRequestHashHex)]; snprintf(stringToSign, sizeof(stringToSign), "AWS4-HMAC-SHA256\n%s\n%s\n%s", - dateStr, scope, canonicalRequestHashHex); + values->requestDateISO8601, scope, canonicalRequestHashHex); #ifdef SIGNATURE_DEBUG printf("--\nString to Sign:\n%s\n", stringToSign); @@ -929,7 +968,8 @@ static S3Status compose_auth_header(const RequestParams *params, #ifdef __APPLE__ unsigned char dateKey[S3_SHA256_DIGEST_LENGTH]; - CCHmac(kCCHmacAlgSHA256, accessKey, strlen(accessKey), dateStr, 8, dateKey); + CCHmac(kCCHmacAlgSHA256, accessKey, strlen(accessKey), + values->requestDateISO8601, 8, dateKey); unsigned char dateRegionKey[S3_SHA256_DIGEST_LENGTH]; CCHmac(kCCHmacAlgSHA256, dateKey, S3_SHA256_DIGEST_LENGTH, awsRegion, strlen(awsRegion), dateRegionKey); @@ -947,7 +987,7 @@ static S3Status compose_auth_header(const RequestParams *params, const EVP_MD *sha256evp = EVP_sha256(); unsigned char dateKey[S3_SHA256_DIGEST_LENGTH]; HMAC(sha256evp, accessKey, strlen(accessKey), - (const unsigned char*) dateStr, 8, dateKey, + (const unsigned char*) values->requestDateISO8601, 8, dateKey, NULL); unsigned char dateRegionKey[S3_SHA256_DIGEST_LENGTH]; HMAC(sha256evp, dateKey, S3_SHA256_DIGEST_LENGTH, @@ -969,22 +1009,21 @@ static S3Status compose_auth_header(const RequestParams *params, #endif len = 0; - char finalSignatureHex[2 * S3_SHA256_DIGEST_LENGTH + 1]; - finalSignatureHex[0] = '\0'; + values->requestSignatureHex[0] = '\0'; for (i = 0; i < S3_SHA256_DIGEST_LENGTH; i++) { - buf_append(finalSignatureHex, "%02x", finalSignature[i]); + buf_append(values->requestSignatureHex, "%02x", finalSignature[i]); } - char credential[strlen(params->bucketContext.accessKeyId) + 1 + 8 + 1 - + strlen(awsRegion) + 1 + 2 + 1 + 12 + 1]; - snprintf(credential, sizeof(credential), "%s/%.8s/%s/s3/aws4_request", - params->bucketContext.accessKeyId, dateStr, awsRegion); + snprintf(values->authCredential, sizeof(values->authCredential), + "%s/%.8s/%s/s3/aws4_request", params->bucketContext.accessKeyId, + values->requestDateISO8601, awsRegion); snprintf( values->authorizationHeader, sizeof(values->authorizationHeader), "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", - credential, values->signedHeaders, finalSignatureHex); + values->authCredential, values->signedHeaders, + values->requestSignatureHex); #ifdef SIGNATURE_DEBUG printf("--\nAuthorization Header:\n%s\n", values->authorizationHeader); @@ -1393,64 +1432,80 @@ void request_api_deinitialize() } } -void request_perform(const RequestParams *params, S3RequestContext *context) +static S3Status setup_request(const RequestParams *params, + RequestComputedValues *computed, + int forceUnsignedPayload) { - Request *request; S3Status status; - int verifyPeerRequest = verifyPeer; - CURLcode curlstatus; - -#define return_status(status) \ - (*(params->completeCallback))(status, 0, params->callbackData); \ - return - - // These will hold the computed values - RequestComputedValues computed; // Validate the bucket name - if (params->bucketContext.bucketName && - ((status = S3_validate_bucket_name - (params->bucketContext.bucketName, - params->bucketContext.uriStyle)) != S3StatusOK)) { - return_status(status); + if (params->bucketContext.bucketName + && ((status = S3_validate_bucket_name(params->bucketContext.bucketName, + params->bucketContext.uriStyle)) + != S3StatusOK)) { + return status; } - char date[64]; if (1) { time_t now = time(NULL); struct tm gmt; gmtime_r(&now, &gmt); - strftime(date, sizeof(date), "%Y%m%dT%H%M%SZ", &gmt); + strftime(computed->requestDateISO8601, + sizeof(computed->requestDateISO8601), "%Y%m%dT%H%M%SZ", &gmt); } else { - snprintf(date, sizeof(date), "20130524T000000Z"); + snprintf(computed->requestDateISO8601, + sizeof(computed->requestDateISO8601), "20130524T000000Z"); } // Compose the amz headers - if ((status = compose_amz_headers(params, date, &computed)) != S3StatusOK) { - return_status(status); + if ((status = compose_amz_headers(params, forceUnsignedPayload, computed)) + != S3StatusOK) { + return status; } // Compose standard headers - if ((status = compose_standard_headers(params, &computed)) != S3StatusOK) { - return_status(status); + if ((status = compose_standard_headers(params, computed)) != S3StatusOK) { + return status; } // URL encode the key - if ((status = encode_key(params, &computed)) != S3StatusOK) { - return_status(status); + if ((status = encode_key(params, computed)) != S3StatusOK) { + return status; } // Compute the canonicalized amz headers - canonicalize_signature_headers(&computed); + canonicalize_signature_headers(computed); // Compute the canonicalized resource - canonicalize_resource(¶ms->bucketContext, computed.urlEncodedKey, - computed.canonicalURI); - canonicalize_query_string(params, computed.canonicalQueryString); + canonicalize_resource(¶ms->bucketContext, computed->urlEncodedKey, + computed->canonicalURI); + canonicalize_query_string(params->queryParams, params->subResource, + computed->canonicalQueryString); // Compose Authorization header - if ((status = compose_auth_header(params, date, &computed)) != S3StatusOK) { + if ((status = compose_auth_header(params, computed)) != S3StatusOK) { + return status; + } + + return status; +} + +void request_perform(const RequestParams *params, S3RequestContext *context) +{ + Request *request; + S3Status status; + int verifyPeerRequest = verifyPeer; + CURLcode curlstatus; + +#define return_status(status) \ + (*(params->completeCallback))(status, 0, params->callbackData); \ + return + + // These will hold the computed values + RequestComputedValues computed; + + if ((status = setup_request(params, &computed, 0)) != S3StatusOK) { return_status(status); } @@ -1613,11 +1668,12 @@ S3Status request_curl_code_to_status(CURLcode code) S3Status S3_generate_authenticated_query_string (char *buffer, const S3BucketContext *bucketContext, - const char *key, int64_t expires, const char *resource) + const char *key, int64_t expires, const char *resource, + const char *httpMethod) { -#define MAX_EXPIRES (((int64_t) 1 << 31) - 1) - // S3 seems to only accept expiration dates up to the number of seconds - // representably by a signed 32-bit integer + // maximum expiration period is seven days (in seconds) +#define MAX_EXPIRES 604800 + if (expires < 0) { expires = MAX_EXPIRES; } @@ -1625,68 +1681,32 @@ S3Status S3_generate_authenticated_query_string expires = MAX_EXPIRES; } - // xxx todo: rework this so that it can be incorporated into shared code - // with request_perform(). It's really unfortunate that this code is not - // shared with request_perform(). + RequestParams params = + { http_request_method_to_type(httpMethod), *bucketContext, key, NULL, + resource, + NULL, NULL, NULL, 0, 0, NULL, NULL, NULL, 0, NULL, NULL, NULL }; - // URL encode the key - char urlEncodedKey[S3_MAX_KEY_SIZE * 3]; - if (key) { - urlEncode(urlEncodedKey, key, strlen(key)); + RequestComputedValues computed; + S3Status status = setup_request(¶ms, &computed, 1); + if (status != S3StatusOK) { + return status; } - else { - urlEncodedKey[0] = 0; - } - - // Compute canonicalized resource - char canonicalizedResource[MAX_CANONICALIZED_RESOURCE_SIZE]; - canonicalize_resource(bucketContext, urlEncodedKey, - canonicalizedResource); - - // We allow for: - // 17 bytes for HTTP-Verb + \n - // 1 byte for empty Content-MD5 + \n - // 1 byte for empty Content-Type + \n - // 20 bytes for Expires + \n - // 0 bytes for CanonicalizedAmzHeaders - // CanonicalizedResource - char signbuf[17 + 1 + 1 + 1 + 20 + sizeof(canonicalizedResource) + 1]; - int len = 0; - -#define signbuf_append(format, ...) \ - len += snprintf(&(signbuf[len]), sizeof(signbuf) - len, \ - format, __VA_ARGS__) - - signbuf_append("%s\n", "GET"); // HTTP-Verb - signbuf_append("%s\n", ""); // Content-MD5 - signbuf_append("%s\n", ""); // Content-Type - signbuf_append("%llu\n", (unsigned long long) expires); - signbuf_append("%s", canonicalizedResource); - - // Generate an HMAC-SHA-1 of the signbuf - unsigned char hmac[20]; - - HMAC_SHA1(hmac, (unsigned char *) bucketContext->secretAccessKey, - strlen(bucketContext->secretAccessKey), - (unsigned char *) signbuf, len); - - // Now base-64 encode the results - char b64[((20 + 1) * 4) / 3]; - int b64Len = base64Encode(hmac, 20, b64); - - // Now urlEncode that - char signature[sizeof(b64) * 3]; - urlEncode(signature, b64, b64Len); - // Finally, compose the uri, with params: - // ?AWSAccessKeyId=xxx[&Expires=]&Signature=xxx - char queryParams[sizeof("AWSAccessKeyId=") + 20 + - sizeof("&Expires=") + 20 + - sizeof("&Signature=") + sizeof(signature) + 1]; + // Finally, compose the URI, with params + char queryParams[sizeof("X-Amz-Algorithm=AWS4-HMAC-SHA256") + + sizeof("&X-Amz-Credential=") + MAX_CREDENTIAL_SIZE + + sizeof("&X-Amz-Date=") + 16 + sizeof("&X-Amz-Expires=") + 6 + + sizeof("&X-Amz-SignedHeaders=") + 128 + sizeof("&X-Amz-Signature=") + + sizeof(computed.requestSignatureHex) + 1]; - sprintf(queryParams, "AWSAccessKeyId=%s&Expires=%ld&Signature=%s", - bucketContext->accessKeyId, (long) expires, signature); + snprintf(queryParams, sizeof(queryParams), + "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=%s" + "&X-Amz-Date=%s&X-Amz-Expires=%llu" + "&X-Amz-SignedHeaders=%s&X-Amz-Signature=%s", + computed.authCredential, computed.requestDateISO8601, expires, + computed.signedHeaders, computed.requestSignatureHex); return compose_uri(buffer, S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE, - bucketContext, urlEncodedKey, resource, queryParams); + bucketContext, computed.urlEncodedKey, resource, + queryParams); } diff --git a/src/s3.c b/src/s3.c index 7d108af..cfaa240 100644 --- a/src/s3.c +++ b/src/s3.c @@ -64,6 +64,7 @@ static S3Protocol protocolG = S3ProtocolHTTPS; static S3UriStyle uriStyleG = S3UriStylePath; static int retriesG = 5; static int verifyPeerG = 0; +static const char *awsRegionG = NULL; // Environment variables, saved as globals ---------------------------------- @@ -156,6 +157,8 @@ static char putenvBufG[256]; #define TARGET_BUCKET_PREFIX_LEN (sizeof(TARGET_BUCKET_PREFIX) - 1) #define TARGET_PREFIX_PREFIX "targetPrefix=" #define TARGET_PREFIX_PREFIX_LEN (sizeof(TARGET_PREFIX_PREFIX) - 1) +#define HTTP_METHOD_PREFIX "method=" +#define HTTP_METHOD_PREFIX_LEN (sizeof(HTTP_METHOD_PREFIX) - 1) // util ---------------------------------------------------------------------- @@ -164,7 +167,7 @@ static void S3_init() { S3Status status; const char *hostname = getenv("S3_HOSTNAME"); - + if ((status = S3_initialize("s3", verifyPeerG|S3_INIT_ALL, hostname)) != S3StatusOK) { fprintf(stderr, "Failed to initialize libs3: %s\n", @@ -201,13 +204,14 @@ static void usageExit(FILE *out) " -r/--retries : retry retryable failures this number of times\n" " (default is 5)\n" " -v/--verify-peer : verify peer SSL certificate (default is no)\n" +" -g/--region : use for request authorization\n" "\n" " Environment:\n" "\n" " S3_ACCESS_KEY_ID : S3 access key ID (required)\n" " S3_SECRET_ACCESS_KEY : S3 secret access key (required)\n" " S3_HOSTNAME : specify alternative S3 host (optional)\n" -"\n" +"\n" " Commands (with and [optional parameters]) :\n" "\n" " (NOTE: all command parameters take a value and are specified using the\n" @@ -322,6 +326,8 @@ static void usageExit(FILE *out) " [expires] : Expiration date for query string\n" " [resource] : Sub-resource of key for query string, without a\n" " leading '?', for example, \"torrent\"\n" +" [method] : HTTP method for use with the query string\n" +" : (default is \"GET\")" "\n" " listmultiparts : Show multipart uploads\n" " : Bucket multipart uploads belongs to\n" @@ -438,7 +444,7 @@ static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) } memcpy(&(buf->data[buf->size]), data, toCopy); - + buf->size += toCopy, data += toCopy, dataLen -= toCopy; } @@ -742,6 +748,7 @@ static struct option longOptionsG[] = { "show-properties", no_argument, 0, 's' }, { "retries", required_argument, 0, 'r' }, { "verify-peer", no_argument, 0, 'v' }, + { "region", required_argument, 0, 'g' }, { 0, 0, 0, 0 } }; @@ -913,8 +920,8 @@ static void list_service(int allDetails) }; do { - S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, 0, 0, - &listServiceHandler, &data); + S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, + awsRegionG, 0, &listServiceHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -957,7 +964,7 @@ static void test_bucket(int argc, char **argv, int optindex) char locationConstraint[64]; do { S3_test_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, 0, - 0, bucketName, 0, sizeof(locationConstraint), + 0, bucketName, awsRegionG, sizeof(locationConstraint), locationConstraint, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -1055,9 +1062,9 @@ static void create_bucket(int argc, char **argv, int optindex) }; do { - S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG, 0, - 0, bucketName, 0, cannedAcl, locationConstraint, 0, - &responseHandler, 0); + S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, + bucketName, awsRegionG, cannedAcl, locationConstraint, + 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -1066,7 +1073,7 @@ static void create_bucket(int argc, char **argv, int optindex) else { printError(); } - + S3_deinitialize(); } @@ -1096,7 +1103,7 @@ static void delete_bucket(int argc, char **argv, int optindex) do { S3_delete_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0, 0, bucketName, 0, 0, &responseHandler, 0); + 0, 0, bucketName, awsRegionG, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -1244,7 +1251,7 @@ static void list_bucket(const char *bucketName, const char *prefix, int maxkeys, int allDetails) { S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1254,7 +1261,7 @@ static void list_bucket(const char *bucketName, const char *prefix, accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ListBucketHandler listBucketHandler = @@ -1393,7 +1400,7 @@ typedef struct list_parts_callback_data typedef struct list_upload_callback_data { - char uploadId[1024]; + char uploadId[1024]; } abort_upload_callback_data; static void printListMultipartHeader(int allDetails) @@ -1408,12 +1415,12 @@ static void printListPartsHeader() printf("%-25s %-30s %-30s %-15s", "LastModified", "PartNumber", "ETag", "SIZE"); - + printf("\n"); printf("--------------------- " - " ------------- " + " ------------- " "------------------------------- " - " -----"); + " -----"); printf("\n"); } @@ -1453,7 +1460,7 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker else { data->nextUploadIdMarker[0] = 0; } - + if (uploadsCount && !data->uploadCount) { printListMultipartHeader(data->allDetails); } @@ -1485,8 +1492,8 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker } else { time_t t = (time_t) upload->initiated; - strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", - gmtime(&t)); + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", + gmtime(&t)); printf("%-50s %s %-50s", upload->key, timebuf, upload->uploadId); if (data->allDetails) { printf(" %-34s %-64s %-12s %-64s %-12s", @@ -1513,16 +1520,16 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker static S3Status listPartsCallback(int isTruncated, - const char *nextPartNumberMarker, - const char *initiatorId, - const char *initiatorDisplayName, - const char *ownerId, - const char *ownerDisplayName, - const char *storageClass, - int partsCount, - int handlePartsStart, - const S3ListPart *parts, - void *callbackData) + const char *nextPartNumberMarker, + const char *initiatorId, + const char *initiatorDisplayName, + const char *ownerId, + const char *ownerDisplayName, + const char *storageClass, + int partsCount, + int handlePartsStart, + const S3ListPart *parts, + void *callbackData) { list_parts_callback_data *data = (list_parts_callback_data *) callbackData; @@ -1611,7 +1618,7 @@ static S3Status listPartsCallback(int isTruncated, } } - data->partsCount += partsCount; + data->partsCount += partsCount; return S3StatusOK; } @@ -1662,16 +1669,16 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) else if (!bucketName) { bucketName = param; } - + else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); } } if (bucketName) { - + S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1681,7 +1688,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ListMultipartUploadsHandler listMultipartUploadsHandler = @@ -1785,16 +1792,16 @@ static void list_parts(int argc, char **argv, int optindex) !strcmp(ad, "1")) { allDetails = 1; } - } + } else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); } } if (bucketName) { - + S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1804,7 +1811,7 @@ static void list_parts(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ListPartsHandler listPartsHandler = @@ -1816,11 +1823,11 @@ static void list_parts(int argc, char **argv, int optindex) list_parts_callback_data data; memset(&data, 0, sizeof(list_parts_callback_data)); - if (partnumbermarker != 0) { + if (partnumbermarker != 0) { snprintf(data.nextPartNumberMarker, sizeof(data.nextPartNumberMarker), "%s", partnumbermarker); } - + data.partsCount = 0; data.allDetails = allDetails; data.noPrint = 0; @@ -1849,9 +1856,7 @@ static void list_parts(int argc, char **argv, int optindex) } S3_deinitialize(); - } - } @@ -1862,7 +1867,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) "\n"); return; } - + // Split bucket/key char *slash = argv[optindex]; while (*slash && (*slash != '/')) { @@ -1879,22 +1884,22 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) const char *key = slash; const char *uploadid = 0; while (optindex < argc) { - char *param = argv[optindex++]; + char *param = argv[optindex++]; if (!strncmp(param, UPLOAD_ID_PREFIX, UPLOAD_ID_PREFIX_LEN)) { uploadid = &(param[UPLOAD_ID_PREFIX_LEN]); } else if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { key = &(param[FILENAME_PREFIX_LEN]); - } + } else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); } } if (bucketName) { - + S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1904,7 +1909,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3AbortMultipartUploadHandler abortMultipartUploadHandler = @@ -1924,7 +1929,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) snprintf(data.nextUploadIdMarker, sizeof(data.nextUploadIdMarker), "%s", uploadidmarker); } - + data.uploadCount = 0; data.allDetails = allDetails; */ @@ -1933,7 +1938,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) S3_abort_multipart_upload(&bucketContext, key, uploadid, &abortMultipartUploadHandler); } while (S3_status_is_retryable(statusG) && should_retry()); - + S3_deinitialize(); } } @@ -1958,7 +1963,7 @@ static void delete_object(int argc, char **argv, int optindex) const char *key = slash; S3_init(); - + S3BucketContext bucketContext = { 0, @@ -1968,11 +1973,11 @@ static void delete_object(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler responseHandler = - { + { 0, &responseCompleteCallback }; @@ -2007,7 +2012,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, { put_object_callback_data *data = (put_object_callback_data *) callbackData; - + int ret = 0; if (data->contentLength) { @@ -2030,7 +2035,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, printf("%llu bytes remaining ", (unsigned long long) data->totalContentLength); printf("(%d%% complete) ...\n", - (int) (((data->totalOriginalContentLength - + (int) (((data->totalOriginalContentLength - data->totalContentLength) * 100) / data->totalOriginalContentLength)); } @@ -2068,6 +2073,7 @@ S3Status MultipartResponseProperiesCallback return S3StatusOK; } + static int multipartPutXmlCallback(int bufferSize, char *buffer, void *callbackData) { @@ -2095,7 +2101,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ListPartsHandler listPartsHandler = @@ -2107,7 +2113,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, list_parts_callback_data data; memset(&data, 0, sizeof(list_parts_callback_data)); - + data.partsCount = 0; data.allDetails = 0; data.manager = manager; @@ -2133,10 +2139,11 @@ static int try_get_parts_info(const char *bucketName, const char *key, printError(); return -1; } - + return 0; } + static void put_object(int argc, char **argv, int optindex, const char *srcBucketName, const char *srcKey, unsigned long long srcSize) { @@ -2348,7 +2355,7 @@ static void put_object(int argc, char **argv, int optindex, contentLength; S3_init(); - + S3BucketContext bucketContext = { 0, @@ -2358,7 +2365,7 @@ static void put_object(int argc, char **argv, int optindex, accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3PutProperties putProperties = @@ -2374,7 +2381,7 @@ static void put_object(int argc, char **argv, int optindex, metaProperties, useServerSideEncryption }; - + if (contentLength <= MULTIPART_CHUNK_SIZE) { S3PutObjectHandler putObjectHandler = { @@ -2437,10 +2444,10 @@ static void put_object(int argc, char **argv, int optindex, &multipartPutXmlCallback, 0 }; - + manager.etags = (char **) malloc(sizeof(char *) * totalSeq); manager.next_etags_pos = 0; - + if (uploadId) { manager.upload_id = strdup(uploadId); manager.remaining = contentLength; @@ -2452,7 +2459,7 @@ static void put_object(int argc, char **argv, int optindex, goto clean; } } - + do { S3_initiate_multipart(&bucketContext,key,0, &handler,0, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -2462,7 +2469,7 @@ static void put_object(int argc, char **argv, int optindex, goto clean; } -upload: +upload: todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { memset(&partData, 0, sizeof(MultipartPartData)); @@ -2488,7 +2495,7 @@ static void put_object(int argc, char **argv, int optindex, accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler copyResponseHandler = { &responsePropertiesCallback, &responseCompleteCallback }; @@ -2517,7 +2524,7 @@ static void put_object(int argc, char **argv, int optindex, contentLength -= MULTIPART_CHUNK_SIZE; todoContentLength -= MULTIPART_CHUNK_SIZE; } - + int i; int size = 0; size += growbuffer_append(&(manager.gb), "", @@ -2553,7 +2560,7 @@ static void put_object(int argc, char **argv, int optindex, growbuffer_destroy(manager.gb); free(manager.etags); } - + S3_deinitialize(); } @@ -2623,7 +2630,7 @@ static void copy_object(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ListBucketHandler listBucketHandler = { @@ -2760,7 +2767,6 @@ static void copy_object(int argc, char **argv, int optindex) } } - S3BucketContext bucketContext = { 0, @@ -2770,7 +2776,7 @@ static void copy_object(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3PutProperties putProperties = @@ -2788,7 +2794,7 @@ static void copy_object(int argc, char **argv, int optindex) }; S3ResponseHandler responseHandler = - { + { &responsePropertiesCallback, &responseCompleteCallback }; @@ -2831,7 +2837,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, FILE *outfile = (FILE *) callbackData; size_t wrote = fwrite(buffer, 1, bufferSize, outfile); - + return ((wrote < (size_t) bufferSize) ? S3StatusAbortedByCallback : S3StatusOK); } @@ -2926,7 +2932,7 @@ static void get_object(int argc, char **argv, int optindex) // unmodified outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); } - + if (!outfile) { fprintf(stderr, "\nERROR: Failed to open output file %s: ", filename); @@ -2943,7 +2949,7 @@ static void get_object(int argc, char **argv, int optindex) } S3_init(); - + S3BucketContext bucketContext = { 0, @@ -2953,7 +2959,7 @@ static void get_object(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3GetConditions getConditions = @@ -2993,7 +2999,7 @@ static void head_object(int argc, char **argv, int optindex) fprintf(stderr, "\nERROR: Missing parameter: bucket/key\n"); usageExit(stderr); } - + // Head implies showing response properties showResponsePropertiesG = 1; @@ -3019,7 +3025,7 @@ static void head_object(int argc, char **argv, int optindex) } S3_init(); - + S3BucketContext bucketContext = { 0, @@ -3029,7 +3035,7 @@ static void head_object(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler responseHandler = @@ -3079,6 +3085,7 @@ static void generate_query_string(int argc, char **argv, int optindex) int64_t expires = -1; const char *resource = 0; + const char *httpMethod = "GET"; while (optindex < argc) { char *param = argv[optindex++]; @@ -3093,6 +3100,9 @@ static void generate_query_string(int argc, char **argv, int optindex) else if (!strncmp(param, RESOURCE_PREFIX, RESOURCE_PREFIX_LEN)) { resource = &(param[RESOURCE_PREFIX_LEN]); } + else if (!strncmp(param, HTTP_METHOD_PREFIX, HTTP_METHOD_PREFIX_LEN)) { + httpMethod = &(param[HTTP_METHOD_PREFIX_LEN]); + } else { fprintf(stderr, "\nERROR: Unknown param: %s\n", param); usageExit(stderr); @@ -3100,7 +3110,7 @@ static void generate_query_string(int argc, char **argv, int optindex) } S3_init(); - + S3BucketContext bucketContext = { 0, @@ -3110,14 +3120,14 @@ static void generate_query_string(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; char buffer[S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE]; S3Status status = S3_generate_authenticated_query_string - (buffer, &bucketContext, key, expires, resource); - + (buffer, &bucketContext, key, expires, resource, httpMethod); + if (status != S3StatusOK) { printf("Failed to generate authenticated query string: %s\n", S3_get_status_name(status)); @@ -3182,7 +3192,7 @@ void get_acl(int argc, char **argv, int optindex) // unmodified outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); } - + if (!outfile) { fprintf(stderr, "\nERROR: Failed to open output file %s: ", filename); @@ -3214,7 +3224,7 @@ void get_acl(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler responseHandler = @@ -3357,7 +3367,7 @@ void set_acl(int argc, char **argv, int optindex) aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0; char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE]; char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE]; - + // Parse it int aclGrantCount; S3AclGrant aclGrants[S3_MAX_ACL_GRANT_COUNT]; @@ -3379,7 +3389,7 @@ void set_acl(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler responseHandler = @@ -3392,7 +3402,7 @@ void set_acl(int argc, char **argv, int optindex) S3_set_acl(&bucketContext, key, ownerId, ownerDisplayName, aclGrantCount, aclGrants, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); - + if (statusG != S3StatusOK) { printError(); } @@ -3440,7 +3450,7 @@ void get_logging(int argc, char **argv, int optindex) // unmodified outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); } - + if (!outfile) { fprintf(stderr, "\nERROR: Failed to open output file %s: ", filename); @@ -3473,7 +3483,7 @@ void get_logging(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler responseHandler = @@ -3507,7 +3517,7 @@ void get_logging(int argc, char **argv, int optindex) char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16]; const char *id; - + switch (grant->granteeType) { case S3GranteeTypeAmazonCustomerByEmail: type = "Email"; @@ -3600,7 +3610,7 @@ void set_logging(int argc, char **argv, int optindex) if (targetBucket) { FILE *infile; - + if (filename) { if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) { fprintf(stderr, "\nERROR: Failed to open input file %s: ", @@ -3618,7 +3628,7 @@ void set_logging(int argc, char **argv, int optindex) aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0; char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE]; char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE]; - + // Parse it if (!convert_simple_acl(aclBuf, ownerId, ownerDisplayName, &aclGrantCount, aclGrants)) { @@ -3641,7 +3651,7 @@ void set_logging(int argc, char **argv, int optindex) accessKeyIdG, secretAccessKeyG, 0, - 0 + awsRegionG }; S3ResponseHandler responseHandler = @@ -3655,7 +3665,7 @@ void set_logging(int argc, char **argv, int optindex) targetPrefix, aclGrantCount, aclGrants, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); - + if (statusG != S3StatusOK) { printError(); } @@ -3671,7 +3681,7 @@ int main(int argc, char **argv) // Parse args while (1) { int idx = 0; - int c = getopt_long(argc, argv, "vfhusr:", longOptionsG, &idx); + int c = getopt_long(argc, argv, "vfhusr:g:", longOptionsG, &idx); if (c == -1) { // End of options @@ -3699,11 +3709,14 @@ int main(int argc, char **argv) retriesG += *v - '0'; v++; } + } break; case 'v': verifyPeerG = S3_INIT_VERIFY_PEER; break; - } + case 'g': + awsRegionG = strdup(optarg); + break; default: fprintf(stderr, "\nERROR: Unknown option: -%c\n", c); // Usage exit From 226bc75b3fc0337ae763db24143b6367ca7dc4f7 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 30 Sep 2016 15:00:02 -0600 Subject: [PATCH 09/57] fix query parameter handling in canonicalization; fix echoing in test script --- inc/libs3.h | 6 +-- src/bucket.c | 2 +- src/general.c | 2 +- src/multipart.c | 18 ++++---- src/object.c | 19 ++++---- src/request.c | 117 +++++++++++++++++++++++++++++++++++------------- src/s3.c | 32 +++++++------ test/test.sh | 10 ++--- 8 files changed, 132 insertions(+), 74 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index 6946507..4e678cd 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -1721,9 +1721,9 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, * @param expires gives the number of seconds since Unix epoch for the * expiration date of the request; after this time, the request will * no longer be valid. If this value is negative, the largest - * expiration date possible is used (currently, Jan 19, 2038). + * expiration interval possible is used (one week). * @param resource gives a sub-resource to be fetched for the request, or NULL - * for none. This should be of the form "?", i.e. + * for none. This should be of the form "?", i.e. * "?torrent". * @param httpMethod the HTTP request method that will be used with the * generated query string (e.g. "GET"). @@ -1735,7 +1735,7 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, **/ S3Status S3_generate_authenticated_query_string (char *buffer, const S3BucketContext *bucketContext, - const char *key, int64_t expires, const char *resource, + const char *key, int expires, const char *resource, const char *httpMethod); diff --git a/src/bucket.c b/src/bucket.c index 66c455d..d83e654 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -707,7 +707,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, } simplexml_initialize(&(lbData->simpleXml), &listBucketXmlCallback, lbData); - + lbData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lbData->listBucketCallback = handler->listBucketCallback; diff --git a/src/general.c b/src/general.c index e5f8696..fa55939 100644 --- a/src/general.c +++ b/src/general.c @@ -184,7 +184,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorUnexpectedContent); handlecase(ErrorUnresolvableGrantByEmailAddress); handlecase(ErrorUserKeyMustBeSpecified); - handlecase(ErrorUnknown); + handlecase(ErrorUnknown); handlecase(HttpErrorMovedTemporarily); handlecase(HttpErrorBadRequest); handlecase(HttpErrorForbidden); diff --git a/src/multipart.c b/src/multipart.c index df88a7e..3101d0b 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -191,8 +191,8 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, const char *upload_id, int partContentLength, S3RequestContext *requestContext, void *callbackData) { - char subResource[512]; - snprintf(subResource, 512, "partNumber=%d&uploadId=%s", seq, upload_id); + char queryParams[512]; + snprintf(queryParams, 512, "partNumber=%d&uploadId=%s", seq, upload_id); RequestParams params = { @@ -206,8 +206,8 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, bucketContext->securityToken, // securityToken bucketContext->authRegion }, // authRegion key, // key - 0, // queryParams - subResource, // subResource + queryParams, // queryParams + 0, // subResource 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions @@ -320,9 +320,9 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, S3RequestContext *requestContext, void *callbackData) { - char subResource[512]; - snprintf(subResource, 512, "uploadId=%s", upload_id); - CommitMultiPartData *data = + char queryParams[512]; + snprintf(queryParams, 512, "uploadId=%s", upload_id); + CommitMultiPartData *data = (CommitMultiPartData *) malloc(sizeof(CommitMultiPartData)); data->userdata = callbackData; data->handler = handler; @@ -344,8 +344,8 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, bucketContext->securityToken, // securityToken bucketContext->authRegion }, // authRegion key, // key - 0, // queryParams - subResource, // subResource + queryParams, // queryParams + 0, // subResource 0, // copySourceBucketName 0, // copySourceKey 0, // getConditions diff --git a/src/object.c b/src/object.c index 5e167fc..cc2be6e 100644 --- a/src/object.c +++ b/src/object.c @@ -188,9 +188,10 @@ void S3_copy_object(const S3BucketContext *bucketContext, const char *key, void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, - const char *destinationBucket, const char *destinationKey, - const int partNo, const char *uploadId, - const unsigned long startOffset, const unsigned long count, + const char *destinationBucket, + const char *destinationKey, const int partNo, + const char *uploadId, const unsigned long startOffset, + const unsigned long count, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, @@ -220,11 +221,11 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, string_buffer_initialize(data->lastModified); // If there's a sequence ID > 0 then add a subResource, OTW pass in NULL - char subResource[512]; - char *subRsrc = NULL; + char queryParams[512]; + char *qp = NULL; if (partNo > 0) { - snprintf(subResource, 512, "partNumber=%d&uploadId=%s", partNo, uploadId); - subRsrc = subResource; + snprintf(queryParams, 512, "partNumber=%d&uploadId=%s", partNo, uploadId); + qp = queryParams; } // Set up the RequestParams @@ -241,8 +242,8 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, bucketContext->securityToken, // securityToken bucketContext->authRegion }, // authRegion destinationKey ? destinationKey : key, // key - 0, // queryParams - subRsrc, // subResource + qp, // queryParams + 0, // subResource bucketContext->bucketName, // copySourceBucketName key, // copySourceKey 0, // getConditions diff --git a/src/request.c b/src/request.c index 1897c97..b2b85ad 100644 --- a/src/request.c +++ b/src/request.c @@ -641,26 +641,27 @@ static S3Status encode_key(const RequestParams *params, } -// Simple comparison function for comparing two HTTP header names that are -// embedded within an HTTP header line, returning true if header1 comes -// before header2 alphabetically, false if not -static int headerle(const char *header1, const char *header2) +// Simple comparison function for comparing two "" +// delimited strings, returning true if the key of s1 comes +// before the key of s2 alphabetically, false if not +static int headerle(const char *s1, const char *s2, char delim) { while (1) { - if (*header1 == ':') { - return (*header2 != ':'); + if (*s1 == delim) { + return (*s2 != delim); } - else if (*header2 == ':') { + else if (*s2 == delim) { return 0; } - else if (*header2 < *header1) { + else if (*s2 < *s1) { return 0; } - else if (*header2 > *header1) { + else if (*s2 > *s1) { return 1; } - header1++, header2++; + s1++, s2++; } + return 0; } @@ -672,18 +673,18 @@ static int headerle(const char *header1, const char *header2) // all the string comparisons that would be done "going forward", and thus // only does the necessary string comparisons to move values back into their // sorted position. -static void header_gnome_sort(const char **headers, int size) +static void kv_gnome_sort(const char **values, int size, char delim) { int i = 0, last_highest = 0; while (i < size) { - if ((i == 0) || headerle(headers[i - 1], headers[i])) { + if ((i == 0) || headerle(values[i - 1], values[i], delim)) { i = ++last_highest; } else { - const char *tmp = headers[i]; - headers[i] = headers[i - 1]; - headers[--i] = tmp; + const char *tmp = values[i]; + values[i] = values[i - 1]; + values[--i] = tmp; } } } @@ -711,7 +712,7 @@ static void canonicalize_signature_headers(RequestComputedValues *values) } // Now sort these - header_gnome_sort(sortedHeaders, headerCount); + kv_gnome_sort(sortedHeaders, headerCount, ':'); // Now copy this sorted list into the buffer, all the while: // - folding repeated headers into single lines, and @@ -807,22 +808,52 @@ static void canonicalize_resource(const S3BucketContext *context, #undef append } -// Canonicalize the query string part of the request into a buffer -static void canonicalize_query_string(const char *queryParams, - const char *subResource, char *buffer) + +static void sort_and_urlencode_query_string(const char *queryString, + char *result) { - int len = 0; +#ifdef SIGNATURE_DEBUG + printf("\n--\nsort_and_urlencode\nqueryString: %s\n", queryString); +#endif - *buffer = 0; + unsigned int numParams = 1; + const char *tmp = queryString; + while ((tmp = strchr(tmp, '&')) != NULL) { + numParams++; + tmp++; + } -#define append(str) len += sprintf(&(buffer[len]), "%s", str) + const char* params[numParams]; - if (queryParams && queryParams[0]) { - char appendage[4]; + char tokenized[strlen(queryString) + 1]; + strncpy(tokenized, queryString, strlen(queryString) + 1); + + char *tok = tokenized; + const char *token = NULL; + char *save = NULL; + unsigned int i = 0; + + while ((token = strtok_r(tok, "&", &save)) != NULL) { + tok = NULL; + params[i++] = token; + } + + kv_gnome_sort(params, numParams, '='); + +#ifdef SIGNATURE_DEBUG + for (i = 0; i < numParams; i++) { + printf("%d: %s\n", i, params[i]); + } +#endif + + unsigned int pi = 0; + char appendage[4]; + + for (; pi < numParams; pi++) { + const char *param = params[pi]; int foundEquals = 0; - unsigned long i = 0; - for (; i < strlen(queryParams); i++) { - char c = queryParams[i]; + for (i = 0; i < strlen(param); i++) { + char c = param[i]; if (isalnum(c) || (c == '_') || (c == '-') || (c == '~') || (c == '.')) { appendage[0] = c; @@ -836,15 +867,37 @@ static void canonicalize_query_string(const char *queryParams, else { snprintf(appendage, 4, "%%%02X", c); } - append(appendage); + strncat(result, appendage, strlen(appendage)); } + strncat(result, "&", 1); + } + result[strlen(result) - 1] = '\0'; +} + + +// Canonicalize the query string part of the request into a buffer +static void canonicalize_query_string(const char *queryParams, + const char *subResource, char *buffer) +{ + int len = 0; + + *buffer = 0; + +#define append(str) len += sprintf(&(buffer[len]), "%s", str) + + if (queryParams && queryParams[0]) { + char sorted[strlen(queryParams) * 2]; + sorted[0] = '\0'; + sort_and_urlencode_query_string(queryParams, sorted); + append(sorted); } if (subResource && subResource[0]) { if (queryParams && queryParams[0]) { - append("?"); + append("&"); } append(subResource); + append("="); } #undef append @@ -1028,7 +1081,7 @@ static S3Status compose_auth_header(const RequestParams *params, #ifdef SIGNATURE_DEBUG printf("--\nAuthorization Header:\n%s\n", values->authorizationHeader); printf("--\nAMZ Headers:\n"); - for (int i = 0; i < values->amzHeadersCount; i++) { + for (i = 0; i < values->amzHeadersCount; i++) { printf("%s\n", values->amzHeaders[i]); } #endif @@ -1668,7 +1721,7 @@ S3Status request_curl_code_to_status(CURLcode code) S3Status S3_generate_authenticated_query_string (char *buffer, const S3BucketContext *bucketContext, - const char *key, int64_t expires, const char *resource, + const char *key, int expires, const char *resource, const char *httpMethod) { // maximum expiration period is seven days (in seconds) @@ -1701,7 +1754,7 @@ S3Status S3_generate_authenticated_query_string snprintf(queryParams, sizeof(queryParams), "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=%s" - "&X-Amz-Date=%s&X-Amz-Expires=%llu" + "&X-Amz-Date=%s&X-Amz-Expires=%d" "&X-Amz-SignedHeaders=%s&X-Amz-Signature=%s", computed.authCredential, computed.requestDateISO8601, expires, computed.signedHeaders, computed.requestSignatureHex); diff --git a/src/s3.c b/src/s3.c index cfaa240..3c66641 100644 --- a/src/s3.c +++ b/src/s3.c @@ -807,7 +807,7 @@ static S3Status responsePropertiesCallback // This callback does the same thing for every request type: saves the status // and error stuff in global variables static void responseCompleteCallback(S3Status status, - const S3ErrorDetails *error, + const S3ErrorDetails *error, void *callbackData) { (void) callbackData; @@ -2429,7 +2429,7 @@ static void put_object(int argc, char **argv, int optindex, &responsePropertiesCallback, &responseCompleteCallback }, - &initial_multipart_callback + &initial_multipart_callback }; S3PutObjectHandler putObjectHandler = { @@ -2439,7 +2439,7 @@ static void put_object(int argc, char **argv, int optindex, S3MultipartCommitHandler commit_handler = { { - &responsePropertiesCallback,&responseCompleteCallback + &responsePropertiesCallback,&responseCompleteCallback }, &multipartPutXmlCallback, 0 @@ -2451,17 +2451,17 @@ static void put_object(int argc, char **argv, int optindex, if (uploadId) { manager.upload_id = strdup(uploadId); manager.remaining = contentLength; - if(!try_get_parts_info(bucketName, key, &manager)) { + if (!try_get_parts_info(bucketName, key, &manager)) { fseek(data.infile, -(manager.remaining), 2); contentLength = manager.remaining; goto upload; - }else { + } else { goto clean; } } do { - S3_initiate_multipart(&bucketContext,key,0, &handler,0, &manager); + S3_initiate_multipart(&bucketContext, key,0, &handler,0, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); if (manager.upload_id == 0 || statusG != S3StatusOK) { @@ -2505,12 +2505,14 @@ static void put_object(int argc, char **argv, int optindex, unsigned long long count = partContentLength - 1; // Inclusive for copies // The default copy callback tries to set this for us, need to allocate here manager.etags[seq-1] = malloc(512); // TBD - magic #! Isa there a max etag defined? - S3_copy_object_range(&srcBucketContext, srcKey, bucketName, key, - seq, manager.upload_id, - startOffset, count, - &putProperties, - &lastModified, 512 /*TBD - magic # */, manager.etags[seq-1], 0, - ©ResponseHandler, 0); + S3_copy_object_range(&srcBucketContext, srcKey, + bucketName, key, + seq, manager.upload_id, + startOffset, count, + &putProperties, + &lastModified, 512 /*TBD - magic # */, + manager.etags[seq-1], 0, + ©ResponseHandler, 0); } else { S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, @@ -2643,7 +2645,9 @@ static void copy_object(int argc, char **argv, int optindex) ".", 1, 0, &listBucketHandler, &sourceSize); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { - fprintf(stderr, "\nERROR: Unable to get source object size\n"); + fprintf(stderr, "\nERROR: Unable to get source object size (%s)\n", + S3_get_status_name(statusG)); + fprintf(stderr, "%s\n", errorDetailsG); exit(1); } if (sourceSize > MULTIPART_CHUNK_SIZE) { @@ -3082,7 +3086,7 @@ static void generate_query_string(int argc, char **argv, int optindex) key = 0; } - int64_t expires = -1; + int expires = -1; const char *resource = 0; const char *httpMethod = "GET"; diff --git a/test/test.sh b/test/test.sh index 4373941..41657da 100755 --- a/test/test.sh +++ b/test/test.sh @@ -106,7 +106,7 @@ COPY_BUCKET=${TEST_BUCKET_PREFIX}.copybucket echo "$S3_COMMAND create $COPY_BUCKET" $S3_COMMAND create $COPY_BUCKET failures=$(($failures + (($? == 0) ? 0 : 1))) -echo <> acl +cat <> acl Group Authenticated AWS Users READ EOF -echo <> acl +cat <> acl Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET filename=acl" @@ -182,10 +182,10 @@ $S3_COMMAND getacl $TEST_BUCKET/aclkey filename=acl failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone -echo <> acl +cat <> acl Group Authenticated AWS Users READ EOF -echo <> acl +cat <> acl Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl" From 584bf483e778a469777c2fbf72fa89a8e05b658d Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Fri, 30 Sep 2016 17:09:18 -0600 Subject: [PATCH 10/57] remove obsolete hash functions, update library major version due to API change --- GNUmakefile | 2 +- GNUmakefile.mingw | 2 +- GNUmakefile.osx | 2 +- inc/util.h | 13 -- src/util.c | 391 +--------------------------------------------- 5 files changed, 5 insertions(+), 405 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index ae0734e..ed40733 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -38,7 +38,7 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 2 +LIBS3_VER_MAJOR ?= 3 LIBS3_VER_MINOR ?= 0 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) diff --git a/GNUmakefile.mingw b/GNUmakefile.mingw index f9ff2ba..b2712f4 100644 --- a/GNUmakefile.mingw +++ b/GNUmakefile.mingw @@ -38,7 +38,7 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 2 +LIBS3_VER_MAJOR ?= 3 LIBS3_VER_MINOR ?= 0 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) diff --git a/GNUmakefile.osx b/GNUmakefile.osx index ab1aff2..63d11d8 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -38,7 +38,7 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 2 +LIBS3_VER_MAJOR ?= 3 LIBS3_VER_MINOR ?= 0 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) diff --git a/inc/util.h b/inc/util.h index f63eaf3..9a65229 100644 --- a/inc/util.h +++ b/inc/util.h @@ -83,19 +83,6 @@ int64_t parseIso8601Time(const char *str); uint64_t parseUnsignedInt(const char *str); -// base64 encode bytes. The output buffer must have at least -// ((4 * (inLen + 1)) / 3) bytes in it. Returns the number of bytes written -// to [out]. -int base64Encode(const unsigned char *in, int inLen, char *out); - -// Compute HMAC-SHA-1 with key [key] and message [message], storing result -// in [hmac] -void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len, - const unsigned char *message, int message_len); - -// Compute a 64-bit hash values given a set of bytes -uint64_t hash(const unsigned char *k, int length); - // Because Windows seems to be missing isblank(), use our own; it's a very // easy function to write in any case int is_blank(char c); diff --git a/src/util.c b/src/util.c index a9d2bc7..bb2ca97 100644 --- a/src/util.c +++ b/src/util.c @@ -120,7 +120,7 @@ int64_t parseIso8601Time(const char *str) str += 2; stm.tm_isdst = -1; - + int64_t ret = mktime(&stm); // Skip the millis @@ -131,7 +131,7 @@ int64_t parseIso8601Time(const char *str) str++; } } - + if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) { int sign = (*str++ == '-') ? -1 : 1; int hours = nextnum(); @@ -164,393 +164,6 @@ uint64_t parseUnsignedInt(const char *str) } -int base64Encode(const unsigned char *in, int inLen, char *out) -{ - static const char *ENC = - "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - - char *original_out = out; - - while (inLen) { - // first 6 bits of char 1 - *out++ = ENC[*in >> 2]; - if (!--inLen) { - // last 2 bits of char 1, 4 bits of 0 - *out++ = ENC[(*in & 0x3) << 4]; - *out++ = '='; - *out++ = '='; - break; - } - // last 2 bits of char 1, first 4 bits of char 2 - *out++ = ENC[((*in & 0x3) << 4) | (*(in + 1) >> 4)]; - in++; - if (!--inLen) { - // last 4 bits of char 2, 2 bits of 0 - *out++ = ENC[(*in & 0xF) << 2]; - *out++ = '='; - break; - } - // last 4 bits of char 2, first 2 bits of char 3 - *out++ = ENC[((*in & 0xF) << 2) | (*(in + 1) >> 6)]; - in++; - // last 6 bits of char 3 - *out++ = ENC[*in & 0x3F]; - in++, inLen--; - } - - return (out - original_out); -} - - -#define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) - -#define blk0L(i) (block->l[i] = (rol(block->l[i], 24) & 0xFF00FF00) \ - | (rol(block->l[i], 8) & 0x00FF00FF)) - -#define blk0B(i) (block->l[i]) - -#define blk(i) (block->l[i & 15] = rol(block->l[(i + 13) & 15] ^ \ - block->l[(i + 8) & 15] ^ \ - block->l[(i + 2) & 15] ^ \ - block->l[i & 15], 1)) - -#define R0_L(v, w, x, y, z, i) \ - z += ((w & (x ^ y)) ^ y) + blk0L(i) + 0x5A827999 + rol(v, 5); \ - w = rol(w, 30); -#define R0_B(v, w, x, y, z, i) \ - z += ((w & (x ^ y)) ^ y) + blk0B(i) + 0x5A827999 + rol(v, 5); \ - w = rol(w, 30); -#define R1(v, w, x, y, z, i) \ - z += ((w & (x ^ y)) ^ y) + blk(i) + 0x5A827999 + rol(v, 5); \ - w = rol(w, 30); -#define R2(v, w, x, y, z, i) \ - z += (w ^ x ^ y) + blk(i) + 0x6ED9EBA1 + rol(v, 5); \ - w = rol(w, 30); -#define R3(v, w, x, y, z, i) \ - z += (((w | x) & y) | (w & x)) + blk(i) + 0x8F1BBCDC + rol(v, 5); \ - w = rol(w, 30); -#define R4(v, w, x, y, z, i) \ - z += (w ^ x ^ y) + blk(i) + 0xCA62C1D6 + rol(v, 5); \ - w = rol(w, 30); - -#define R0A_L(i) R0_L(a, b, c, d, e, i) -#define R0B_L(i) R0_L(b, c, d, e, a, i) -#define R0C_L(i) R0_L(c, d, e, a, b, i) -#define R0D_L(i) R0_L(d, e, a, b, c, i) -#define R0E_L(i) R0_L(e, a, b, c, d, i) - -#define R0A_B(i) R0_B(a, b, c, d, e, i) -#define R0B_B(i) R0_B(b, c, d, e, a, i) -#define R0C_B(i) R0_B(c, d, e, a, b, i) -#define R0D_B(i) R0_B(d, e, a, b, c, i) -#define R0E_B(i) R0_B(e, a, b, c, d, i) - -#define R1A(i) R1(a, b, c, d, e, i) -#define R1B(i) R1(b, c, d, e, a, i) -#define R1C(i) R1(c, d, e, a, b, i) -#define R1D(i) R1(d, e, a, b, c, i) -#define R1E(i) R1(e, a, b, c, d, i) - -#define R2A(i) R2(a, b, c, d, e, i) -#define R2B(i) R2(b, c, d, e, a, i) -#define R2C(i) R2(c, d, e, a, b, i) -#define R2D(i) R2(d, e, a, b, c, i) -#define R2E(i) R2(e, a, b, c, d, i) - -#define R3A(i) R3(a, b, c, d, e, i) -#define R3B(i) R3(b, c, d, e, a, i) -#define R3C(i) R3(c, d, e, a, b, i) -#define R3D(i) R3(d, e, a, b, c, i) -#define R3E(i) R3(e, a, b, c, d, i) - -#define R4A(i) R4(a, b, c, d, e, i) -#define R4B(i) R4(b, c, d, e, a, i) -#define R4C(i) R4(c, d, e, a, b, i) -#define R4D(i) R4(d, e, a, b, c, i) -#define R4E(i) R4(e, a, b, c, d, i) - - -static void SHA1_transform(uint32_t state[5], const unsigned char buffer[64]) -{ - uint32_t a, b, c, d, e; - - typedef union { - unsigned char c[64]; - uint32_t l[16]; - } u; - - unsigned char w[64]; - u *block = (u *) w; - - memcpy(block, buffer, 64); - - a = state[0]; - b = state[1]; - c = state[2]; - d = state[3]; - e = state[4]; - - static uint32_t endianness_indicator = 0x1; - if (((unsigned char *) &endianness_indicator)[0]) { - R0A_L( 0); - R0E_L( 1); R0D_L( 2); R0C_L( 3); R0B_L( 4); R0A_L( 5); - R0E_L( 6); R0D_L( 7); R0C_L( 8); R0B_L( 9); R0A_L(10); - R0E_L(11); R0D_L(12); R0C_L(13); R0B_L(14); R0A_L(15); - } - else { - R0A_B( 0); - R0E_B( 1); R0D_B( 2); R0C_B( 3); R0B_B( 4); R0A_B( 5); - R0E_B( 6); R0D_B( 7); R0C_B( 8); R0B_B( 9); R0A_B(10); - R0E_B(11); R0D_B(12); R0C_B(13); R0B_B(14); R0A_B(15); - } - R1E(16); R1D(17); R1C(18); R1B(19); R2A(20); - R2E(21); R2D(22); R2C(23); R2B(24); R2A(25); - R2E(26); R2D(27); R2C(28); R2B(29); R2A(30); - R2E(31); R2D(32); R2C(33); R2B(34); R2A(35); - R2E(36); R2D(37); R2C(38); R2B(39); R3A(40); - R3E(41); R3D(42); R3C(43); R3B(44); R3A(45); - R3E(46); R3D(47); R3C(48); R3B(49); R3A(50); - R3E(51); R3D(52); R3C(53); R3B(54); R3A(55); - R3E(56); R3D(57); R3C(58); R3B(59); R4A(60); - R4E(61); R4D(62); R4C(63); R4B(64); R4A(65); - R4E(66); R4D(67); R4C(68); R4B(69); R4A(70); - R4E(71); R4D(72); R4C(73); R4B(74); R4A(75); - R4E(76); R4D(77); R4C(78); R4B(79); - - state[0] += a; - state[1] += b; - state[2] += c; - state[3] += d; - state[4] += e; -} - - -typedef struct -{ - uint32_t state[5]; - uint32_t count[2]; - unsigned char buffer[64]; -} SHA1Context; - - -static void SHA1_init(SHA1Context *context) -{ - context->state[0] = 0x67452301; - context->state[1] = 0xEFCDAB89; - context->state[2] = 0x98BADCFE; - context->state[3] = 0x10325476; - context->state[4] = 0xC3D2E1F0; - context->count[0] = context->count[1] = 0; -} - - -static void SHA1_update(SHA1Context *context, const unsigned char *data, - unsigned int len) -{ - uint32_t i, j; - - j = (context->count[0] >> 3) & 63; - - if ((context->count[0] += len << 3) < (len << 3)) { - context->count[1]++; - } - - context->count[1] += (len >> 29); - - if ((j + len) > 63) { - memcpy(&(context->buffer[j]), data, (i = 64 - j)); - SHA1_transform(context->state, context->buffer); - for ( ; (i + 63) < len; i += 64) { - SHA1_transform(context->state, &(data[i])); - } - j = 0; - } - else { - i = 0; - } - - memcpy(&(context->buffer[j]), &(data[i]), len - i); -} - - -static void SHA1_final(unsigned char digest[20], SHA1Context *context) -{ - uint32_t i; - unsigned char finalcount[8]; - - for (i = 0; i < 8; i++) { - finalcount[i] = (unsigned char) - ((context->count[(i >= 4 ? 0 : 1)] >> - ((3 - (i & 3)) * 8)) & 255); - } - - SHA1_update(context, (unsigned char *) "\200", 1); - - while ((context->count[0] & 504) != 448) { - SHA1_update(context, (unsigned char *) "\0", 1); - } - - SHA1_update(context, finalcount, 8); - - for (i = 0; i < 20; i++) { - digest[i] = (unsigned char) - ((context->state[i >> 2] >> ((3 - (i & 3)) * 8)) & 255); - } - - memset(context->buffer, 0, 64); - memset(context->state, 0, 20); - memset(context->count, 0, 8); - memset(&finalcount, 0, 8); - - SHA1_transform(context->state, context->buffer); -} - - -// HMAC-SHA-1: -// -// K - is key padded with zeros to 512 bits -// m - is message -// OPAD - 0x5c5c5c... -// IPAD - 0x363636... -// -// HMAC(K,m) = SHA1((K ^ OPAD) . SHA1((K ^ IPAD) . m)) -void HMAC_SHA1(unsigned char hmac[20], const unsigned char *key, int key_len, - const unsigned char *message, int message_len) -{ - unsigned char kopad[64], kipad[64]; - int i; - - if (key_len > 64) { - key_len = 64; - } - - for (i = 0; i < key_len; i++) { - kopad[i] = key[i] ^ 0x5c; - kipad[i] = key[i] ^ 0x36; - } - - for ( ; i < 64; i++) { - kopad[i] = 0 ^ 0x5c; - kipad[i] = 0 ^ 0x36; - } - - unsigned char digest[20]; - - SHA1Context context; - - SHA1_init(&context); - SHA1_update(&context, kipad, 64); - SHA1_update(&context, message, message_len); - SHA1_final(digest, &context); - - SHA1_init(&context); - SHA1_update(&context, kopad, 64); - SHA1_update(&context, digest, 20); - SHA1_final(hmac, &context); -} - -#define rot(x,k) (((x) << (k)) | ((x) >> (32 - (k)))) - -uint64_t hash(const unsigned char *k, int length) -{ - uint32_t a, b, c; - - a = b = c = 0xdeadbeef + ((uint32_t) length); - - static uint32_t endianness_indicator = 0x1; - if (((unsigned char *) &endianness_indicator)[0]) { - while (length > 12) { - a += k[0]; - a += ((uint32_t) k[1]) << 8; - a += ((uint32_t) k[2]) << 16; - a += ((uint32_t) k[3]) << 24; - b += k[4]; - b += ((uint32_t) k[5]) << 8; - b += ((uint32_t) k[6]) << 16; - b += ((uint32_t) k[7]) << 24; - c += k[8]; - c += ((uint32_t) k[9]) << 8; - c += ((uint32_t) k[10]) << 16; - c += ((uint32_t) k[11]) << 24; - a -= c; a ^= rot(c, 4); c += b; - b -= a; b ^= rot(a, 6); a += c; - c -= b; c ^= rot(b, 8); b += a; - a -= c; a ^= rot(c, 16); c += b; - b -= a; b ^= rot(a, 19); a += c; - c -= b; c ^= rot(b, 4); b += a; - length -= 12; - k += 12; - } - - switch(length) { - case 12: c += ((uint32_t) k[11]) << 24; - case 11: c += ((uint32_t) k[10]) << 16; - case 10: c += ((uint32_t) k[9]) << 8; - case 9 : c += k[8]; - case 8 : b += ((uint32_t) k[7]) << 24; - case 7 : b += ((uint32_t) k[6]) << 16; - case 6 : b += ((uint32_t) k[5]) << 8; - case 5 : b += k[4]; - case 4 : a += ((uint32_t) k[3]) << 24; - case 3 : a += ((uint32_t) k[2]) << 16; - case 2 : a += ((uint32_t) k[1]) << 8; - case 1 : a += k[0]; break; - case 0 : goto end; - } - } - else { - while (length > 12) { - a += ((uint32_t) k[0]) << 24; - a += ((uint32_t) k[1]) << 16; - a += ((uint32_t) k[2]) << 8; - a += ((uint32_t) k[3]); - b += ((uint32_t) k[4]) << 24; - b += ((uint32_t) k[5]) << 16; - b += ((uint32_t) k[6]) << 8; - b += ((uint32_t) k[7]); - c += ((uint32_t) k[8]) << 24; - c += ((uint32_t) k[9]) << 16; - c += ((uint32_t) k[10]) << 8; - c += ((uint32_t) k[11]); - a -= c; a ^= rot(c, 4); c += b; - b -= a; b ^= rot(a, 6); a += c; - c -= b; c ^= rot(b, 8); b += a; - a -= c; a ^= rot(c, 16); c += b; - b -= a; b ^= rot(a, 19); a += c; - c -= b; c ^= rot(b, 4); b += a; - length -= 12; - k += 12; - } - - switch(length) { - case 12: c += k[11]; - case 11: c += ((uint32_t) k[10]) << 8; - case 10: c += ((uint32_t) k[9]) << 16; - case 9 : c += ((uint32_t) k[8]) << 24; - case 8 : b += k[7]; - case 7 : b += ((uint32_t) k[6]) << 8; - case 6 : b += ((uint32_t) k[5]) << 16; - case 5 : b += ((uint32_t) k[4]) << 24; - case 4 : a += k[3]; - case 3 : a += ((uint32_t) k[2]) << 8; - case 2 : a += ((uint32_t) k[1]) << 16; - case 1 : a += ((uint32_t) k[0]) << 24; break; - case 0 : goto end; - } - } - - c ^= b; c -= rot(b, 14); - a ^= c; a -= rot(c, 11); - b ^= a; b -= rot(a, 25); - c ^= b; c -= rot(b, 16); - a ^= c; a -= rot(c, 4); - b ^= a; b -= rot(a, 14); - c ^= b; c -= rot(b, 24); - - end: - return ((((uint64_t) c) << 32) | b); -} - int is_blank(char c) { return ((c == ' ') || (c == '\t')); From e466cc7f516262e368c029e5be1c63ce4e77c22a Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Sun, 2 Oct 2016 11:42:05 -0600 Subject: [PATCH 11/57] fix expected output for ACL with new identifier format --- test/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test.sh b/test/test.sh index 41657da..b67ab40 100755 --- a/test/test.sh +++ b/test/test.sh @@ -157,10 +157,10 @@ failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone cat <> acl -Group Authenticated AWS Users READ +Group Authenticated AWS Users READ EOF cat <> acl -Group All Users READ_ACP +Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET filename=acl" $S3_COMMAND setacl $TEST_BUCKET filename=acl @@ -183,10 +183,10 @@ failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone cat <> acl -Group Authenticated AWS Users READ +Group Authenticated AWS Users READ EOF cat <> acl -Group All Users READ_ACP +Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl" $S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl From c3ed9e52073c7649207d639eb398f0a26d3784a6 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Sun, 2 Oct 2016 16:24:42 -0600 Subject: [PATCH 12/57] missed some whitespace in previous commit --- test/test.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/test.sh b/test/test.sh index b67ab40..413554b 100755 --- a/test/test.sh +++ b/test/test.sh @@ -157,10 +157,10 @@ failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone cat <> acl -Group Authenticated AWS Users READ +Group Authenticated AWS Users READ EOF cat <> acl -Group All Users READ_ACP +Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET filename=acl" $S3_COMMAND setacl $TEST_BUCKET filename=acl @@ -183,10 +183,10 @@ failures=$(($failures + (($? == 0) ? 0 : 1))) # Add READ for all AWS users, and READ_ACP for everyone cat <> acl -Group Authenticated AWS Users READ +Group Authenticated AWS Users READ EOF cat <> acl -Group All Users READ_ACP +Group All Users READ_ACP EOF echo "$S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl" $S3_COMMAND setacl $TEST_BUCKET/aclkey filename=acl From 20785ed806e816003d945dc1a408674398e95bee Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 3 Oct 2016 08:47:23 -0600 Subject: [PATCH 13/57] fix uninitialized memory issue --- src/request.c | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/request.c b/src/request.c index b2b85ad..6ab528c 100644 --- a/src/request.c +++ b/src/request.c @@ -304,7 +304,7 @@ static S3Status append_amz_header(RequestComputedValues *values, values->amzHeadersRaw[rawPos++] = tolower(headerStr[i]); } - strcat(&(values->amzHeadersRaw[rawPos]), ": "); + snprintf(&(values->amzHeadersRaw[rawPos]), 3, ": "); rawPos += 2; for (i = 0; i < strlen(headerValue); i++) { @@ -1080,10 +1080,6 @@ static S3Status compose_auth_header(const RequestParams *params, #ifdef SIGNATURE_DEBUG printf("--\nAuthorization Header:\n%s\n", values->authorizationHeader); - printf("--\nAMZ Headers:\n"); - for (i = 0; i < values->amzHeadersCount; i++) { - printf("%s\n", values->amzHeaders[i]); - } #endif return S3StatusOK; @@ -1541,6 +1537,14 @@ static S3Status setup_request(const RequestParams *params, return status; } +#ifdef SIGNATURE_DEBUG + int i = 0; + printf("\n--\nAMZ Headers:\n"); + for (; i < computed->amzHeadersCount; i++) { + printf("%s\n", computed->amzHeaders[i]); + } +#endif + return status; } From 02f519d993723e59ba4db1985f1dfc72c07a09a7 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 3 Oct 2016 09:06:45 -0600 Subject: [PATCH 14/57] increase length of auth. header to account for requests with many amz headers --- src/request.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/request.c b/src/request.c index 6ab528c..3c9c263 100644 --- a/src/request.c +++ b/src/request.c @@ -125,7 +125,7 @@ typedef struct RequestComputedValues char rangeHeader[128]; // Authorization header - char authorizationHeader[256]; + char authorizationHeader[1024]; // Request date stamp char requestDateISO8601[64]; From 0feed6bec9d1f8bc5b54e98fe3748ef376283dcc Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 4 Oct 2016 15:30:05 -0600 Subject: [PATCH 15/57] clean up test code --- GNUmakefile | 13 ---------- GNUmakefile.osx | 13 ---------- src/request.c | 18 +++++-------- src/testsignature.c | 63 --------------------------------------------- 4 files changed, 6 insertions(+), 101 deletions(-) delete mode 100644 src/testsignature.c diff --git a/GNUmakefile b/GNUmakefile index ed40733..c4ebaad 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -207,14 +207,6 @@ uninstall: # -------------------------------------------------------------------------- # Compile target patterns -$(BUILD)/obj/test%.o: src/test%.c - $(QUIET_ECHO) $@: Compiling test object - @ mkdir -p $(dir $(BUILD)/dep/$<) - @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ - -o $(BUILD)/dep/$(<:%.c=%.d) -c $< - @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) $(CC) -Wno-error=unused-parameter $(CFLAGS) -o $@ -c $< - $(BUILD)/obj/%.o: src/%.c $(QUIET_ECHO) $@: Compiling object @ mkdir -p $(dir $(BUILD)/dep/$<) @@ -293,11 +285,6 @@ $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) @ mkdir -p $(dir $@) $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LIBXML2_LIBS) -$(BUILD)/bin/testsignature: $(BUILD)/obj/testsignature.o $(LIBS3_STATIC) - $(QUIET_ECHO) $@: Building executable - @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) - # -------------------------------------------------------------------------- # Clean target diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 63d11d8..ee051f9 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -195,14 +195,6 @@ uninstall: # -------------------------------------------------------------------------- # Compile target patterns -$(BUILD)/obj/test%.o: src/test%.c - $(QUIET_ECHO) $@: Compiling test object - @ mkdir -p $(dir $(BUILD)/dep/$<) - @ gcc $(CFLAGS) -Wno-error=unused-parameter -M -MG -MQ $@ \ - -DCOMPILINGDEPENDENCIES -o $(BUILD)/dep/$(<:%.c=%.d) -c $< - @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) gcc $(CFLAGS) -Wno-error=unused-parameter -o $@ -c $< - $(BUILD)/obj/%.o: src/%.c $(QUIET_ECHO) $@: Compiling object @ mkdir -p $(dir $(BUILD)/dep/$<) @@ -283,11 +275,6 @@ $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) @ mkdir -p $(dir $@) $(VERBOSE_SHOW) gcc -o $@ $^ $(LIBXML2_LIBS) -$(BUILD)/bin/testsignature: $(BUILD)/obj/testsignature.o $(LIBS3_STATIC) - $(QUIET_ECHO) $@: Building executable - @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) gcc -o $@ $^ $(LDFLAGS) - # -------------------------------------------------------------------------- # Clean target diff --git a/src/request.c b/src/request.c index 3c9c263..a1e8a80 100644 --- a/src/request.c +++ b/src/request.c @@ -134,7 +134,7 @@ typedef struct RequestComputedValues char authCredential[MAX_CREDENTIAL_SIZE + 1]; // Computed request signature (hex string) - char requestSignatureHex[2 * S3_SHA256_DIGEST_LENGTH + 1]; + char requestSignatureHex[S3_SHA256_DIGEST_LENGTH * 2 + 1]; // Host header char hostHeader[128]; @@ -1495,17 +1495,11 @@ static S3Status setup_request(const RequestParams *params, return status; } - if (1) { - time_t now = time(NULL); - struct tm gmt; - gmtime_r(&now, &gmt); - strftime(computed->requestDateISO8601, - sizeof(computed->requestDateISO8601), "%Y%m%dT%H%M%SZ", &gmt); - } - else { - snprintf(computed->requestDateISO8601, - sizeof(computed->requestDateISO8601), "20130524T000000Z"); - } + time_t now = time(NULL); + struct tm gmt; + gmtime_r(&now, &gmt); + strftime(computed->requestDateISO8601, sizeof(computed->requestDateISO8601), + "%Y%m%dT%H%M%SZ", &gmt); // Compose the amz headers if ((status = compose_amz_headers(params, forceUnsignedPayload, computed)) diff --git a/src/testsignature.c b/src/testsignature.c deleted file mode 100644 index f80e6da..0000000 --- a/src/testsignature.c +++ /dev/null @@ -1,63 +0,0 @@ -/* - * testsignature.c - * - * Created on: Sep 29, 2016 - * Author: eric - */ - -#include -#include -#include - -static S3Status responsePropertiesCallback( - const S3ResponseProperties *properties, void *callbackData) -{ - return S3StatusOK; -} - -static void responseCompleteCallback(S3Status status, - const S3ErrorDetails *error, - void *callbackData) -{ - if (error) { - printf("\nERROR\n"); - if (error->message) { - printf("%s\n", error->message); - } - if (error->extraDetailsCount > 0) { - int i = 0; - for (; i < error->extraDetailsCount; i++) { - printf("%s: %s\n", error->extraDetails[i].name, - error->extraDetails[i].value); - } - } - } -} - -static S3Status getObjectDataCallback(int bufferSize, const char *buffer, - void *callbackData) -{ - return S3StatusOK; -} - -int main(int argc, char **argv) -{ - S3_initialize(NULL, S3_INIT_ALL, NULL); - - S3BucketContext bucketContext = - { 0, "examplebucket", S3ProtocolHTTPS, S3UriStyleVirtualHost, - "AKIAIOSFODNN7EXAMPLE", "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", - 0, 0 }; - - S3GetObjectHandler getObjectHandler = - { - { &responsePropertiesCallback, &responseCompleteCallback }, - &getObjectDataCallback }; - - S3_get_object(&bucketContext, "test.txt", NULL, 0, 10, NULL, - &getObjectHandler, NULL); - - S3_deinitialize(); - - return 0; -} From 68d30d6419de3a1fcb1e7136876d02160250f566 Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Tue, 4 Oct 2016 15:37:19 -0600 Subject: [PATCH 16/57] remove temporary test drivers from makefiles --- GNUmakefile | 2 +- GNUmakefile.osx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index c4ebaad..ab64908 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -278,7 +278,7 @@ $(BUILD)/include/libs3.h: inc/libs3.h # Test targets .PHONY: test -test: $(BUILD)/bin/testsimplexml $(BUILD)/bin/testsignature +test: $(BUILD)/bin/testsimplexml $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) $(QUIET_ECHO) $@: Building executable diff --git a/GNUmakefile.osx b/GNUmakefile.osx index ee051f9..103615a 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -268,7 +268,7 @@ $(BUILD)/include/libs3.h: inc/libs3.h # Test targets .PHONY: test -test: $(BUILD)/bin/testsimplexml $(BUILD)/bin/testsignature +test: $(BUILD)/bin/testsimplexml $(BUILD)/bin/testsimplexml: $(BUILD)/obj/testsimplexml.o $(LIBS3_STATIC) $(QUIET_ECHO) $@: Building executable From 61f65659f1f821d4fe8235eee8583a3da88e244d Mon Sep 17 00:00:00 2001 From: Eric Stadtherr Date: Mon, 10 Oct 2016 07:35:45 -0600 Subject: [PATCH 17/57] fix the logic that determines the hostname to use in HTTP headers --- src/bucket.c | 2 +- src/request.c | 19 ++++++------------- src/s3.c | 3 ++- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/src/bucket.c b/src/bucket.c index d83e654..5f998ab 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -645,7 +645,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, // Compose the query params string_buffer(queryParams, 4096); string_buffer_initialize(queryParams); - + #define safe_append(name, value) \ do { \ int fit; \ diff --git a/src/request.c b/src/request.c index a1e8a80..318d1f5 100644 --- a/src/request.c +++ b/src/request.c @@ -517,20 +517,13 @@ static S3Status compose_standard_headers(const RequestParams *params, } values->hostHeader[len] = 0; } - else if (params->bucketContext.hostName) { - size_t len = snprintf(values->hostHeader, sizeof(values->hostHeader), - "Host: %s", params->bucketContext.hostName); - if (len >= sizeof(values->hostHeader)) { - return S3StatusUriTooLong; - } - while (is_blank(values->hostHeader[len])) { - len--; - } - values->hostHeader[len] = 0; - } else { - size_t len = snprintf(values->hostHeader, sizeof(values->hostHeader), - "Host: %s", S3_DEFAULT_HOSTNAME); + size_t len = snprintf( + values->hostHeader, + sizeof(values->hostHeader), + "Host: %s", + params->bucketContext.hostName ? + params->bucketContext.hostName : defaultHostNameG); if (len >= sizeof(values->hostHeader)) { return S3StatusUriTooLong; } diff --git a/src/s3.c b/src/s3.c index 3c66641..797a124 100644 --- a/src/s3.c +++ b/src/s3.c @@ -1317,6 +1317,7 @@ static void list(int argc, char **argv, int optindex) int maxkeys = 0, allDetails = 0; while (optindex < argc) { char *param = argv[optindex++]; + if (!strncmp(param, PREFIX_PREFIX, PREFIX_PREFIX_LEN)) { prefix = &(param[PREFIX_PREFIX_LEN]); } @@ -1700,7 +1701,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) list_multiparts_callback_data data; memset(&data, 0, sizeof(list_multiparts_callback_data)); - if (keymarker != 0) { + if (keymarker != 0) { snprintf(data.nextKeyMarker, sizeof(data.nextKeyMarker), "%s", keymarker); } From da6b030090abe37e794b13ab280fbd7f626dbbf6 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Thu, 27 Oct 2016 11:12:18 +0300 Subject: [PATCH 18/57] headers: do not append empty Transfer-Encoding header Empty header is not sent by curl, but gets included to the list of signed headers and breaks authorization for minio. And probably for other server implementations too. --- src/request.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/request.c b/src/request.c index 318d1f5..d6befc0 100644 --- a/src/request.c +++ b/src/request.c @@ -1231,14 +1231,8 @@ static S3Status setup_curl(Request *request, snprintf(header, sizeof(header), "Content-Length: %llu", (unsigned long long) params->toS3CallbackTotalSize); request->headers = curl_slist_append(request->headers, header); - request->headers = curl_slist_append(request->headers, - "Transfer-Encoding:"); } - else if (params->httpRequestType == HttpRequestTypeCOPY) { - request->headers = curl_slist_append(request->headers, - "Transfer-Encoding:"); - } - + append_standard_header(hostHeader); append_standard_header(cacheControlHeader); append_standard_header(contentTypeHeader); From 449c4a477454d3cc2177259d1806299c808bf839 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Thu, 27 Oct 2016 11:12:18 +0300 Subject: [PATCH 19/57] headers: do not append empty Transfer-Encoding header Empty header is not sent by curl, but gets included to the list of signed headers and breaks authorization for minio. And probably for other server implementations too. --- src/request.c | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/request.c b/src/request.c index df871b2..7d41477 100644 --- a/src/request.c +++ b/src/request.c @@ -936,14 +936,8 @@ static S3Status setup_curl(Request *request, snprintf(header, sizeof(header), "Content-Length: %llu", (unsigned long long) params->toS3CallbackTotalSize); request->headers = curl_slist_append(request->headers, header); - request->headers = curl_slist_append(request->headers, - "Transfer-Encoding:"); } - else if (params->httpRequestType == HttpRequestTypeCOPY) { - request->headers = curl_slist_append(request->headers, - "Transfer-Encoding:"); - } - + append_standard_header(hostHeader); append_standard_header(cacheControlHeader); append_standard_header(contentTypeHeader); From a052a008e25be0872ca30b307d60186812156ed1 Mon Sep 17 00:00:00 2001 From: Bryan Ischo Date: Fri, 4 Nov 2016 09:02:53 -0700 Subject: [PATCH 20/57] Revert prior change, which appears to have broken S3 authentication. --- src/request.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/request.c b/src/request.c index efb31db..000e7af 100644 --- a/src/request.c +++ b/src/request.c @@ -936,8 +936,14 @@ static S3Status setup_curl(Request *request, snprintf(header, sizeof(header), "Content-Length: %llu", (unsigned long long) params->toS3CallbackTotalSize); request->headers = curl_slist_append(request->headers, header); + request->headers = curl_slist_append(request->headers, + "Transfer-Encoding:"); } - + else if (params->httpRequestType == HttpRequestTypeCOPY) { + request->headers = curl_slist_append(request->headers, + "Transfer-Encoding:"); + } + append_standard_header(hostHeader); append_standard_header(cacheControlHeader); append_standard_header(contentTypeHeader); From 7458769658aac1e0f6f1f2d336928c3344600ce2 Mon Sep 17 00:00:00 2001 From: Timo Bingmann Date: Tue, 8 Nov 2016 13:41:05 +0100 Subject: [PATCH 21/57] fix S3_destroy_request_context() to correctly abort curl_multi context --- src/request_context.c | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/request_context.c b/src/request_context.c index d65a07e..cf82f2d 100644 --- a/src/request_context.c +++ b/src/request_context.c @@ -55,19 +55,21 @@ S3Status S3_create_request_context(S3RequestContext **requestContextReturn) void S3_destroy_request_context(S3RequestContext *requestContext) { - curl_multi_cleanup(requestContext->curlm); - - // For each request in the context, call back its done method with - // 'interrupted' status + // For each request in the context, remove curl handle, call back its done + // method with 'interrupted' status Request *r = requestContext->requests, *rFirst = r; if (r) do { r->status = S3StatusInterrupted; + // remove easy handle from a multi session + curl_multi_remove_handle(requestContext->curlm, r->curl); Request *rNext = r->next; request_finish(r); r = rNext; } while (r != rFirst); + curl_multi_cleanup(requestContext->curlm); + free(requestContext); } From 1814918ab2803a6b6747fd2444cc466a7dd79250 Mon Sep 17 00:00:00 2001 From: Joseph Bester Date: Tue, 6 Dec 2016 10:54:13 -0500 Subject: [PATCH 22/57] Isolate xml parsing fix --- src/bucket.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/bucket.c b/src/bucket.c index dcbcf2f..a5daa9f 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -537,10 +537,11 @@ static S3Status listBucketXmlCallback(const char *elementPath, else if (!strcmp(elementPath, "ListBucketResult/CommonPrefixes/Prefix")) { int which = lbData->commonPrefixesCount; + size_t oldLen = lbData->commonPrefixLens[which]; lbData->commonPrefixLens[which] += - snprintf(lbData->commonPrefixes[which], + snprintf(lbData->commonPrefixes[which]+oldLen, sizeof(lbData->commonPrefixes[which]) - - lbData->commonPrefixLens[which] - 1, + oldLen - 1, "%.*s", dataLen, data); if (lbData->commonPrefixLens[which] >= (int) sizeof(lbData->commonPrefixes[which])) { From dc82436e34d02088fe43eef592c40d47bd1ee052 Mon Sep 17 00:00:00 2001 From: Bryan Ischo Date: Wed, 14 Dec 2016 10:08:57 -0800 Subject: [PATCH 23/57] Added comment describing osx makefile workaround. --- GNUmakefile.osx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 103615a..5568a0a 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -102,6 +102,8 @@ endif ifndef LIBXML2_LIBS LIBXML2_LIBS := $(shell xml2-config --libs) + # Work around missing libsystem_symptoms.dylib in Xcode 8; see + # http://stackoverflow.com/questions/39536144/libsystem-symptoms-dylib-missing-in-xcode-8 LIBXML2_LIBS := $(filter-out -L$(shell xcrun --show-sdk-path)/usr/lib, $(LIBXML2_LIBS)) endif From bb96e59583266a7abc9be7fc5d4d4f0e9c1167cb Mon Sep 17 00:00:00 2001 From: Bryan Ischo Date: Wed, 14 Dec 2016 10:17:36 -0800 Subject: [PATCH 24/57] Minor whitespace fix to satisfy my OCD. --- GNUmakefile.osx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 5568a0a..25def23 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -103,7 +103,7 @@ endif ifndef LIBXML2_LIBS LIBXML2_LIBS := $(shell xml2-config --libs) # Work around missing libsystem_symptoms.dylib in Xcode 8; see - # http://stackoverflow.com/questions/39536144/libsystem-symptoms-dylib-missing-in-xcode-8 + # http://stackoverflow.com/questions/39536144/libsystem-symptoms-dylib-missing-in-xcode-8 LIBXML2_LIBS := $(filter-out -L$(shell xcrun --show-sdk-path)/usr/lib, $(LIBXML2_LIBS)) endif From e5ff0c7b0067b0303cabd25ddcdb2879c31ebd01 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Wed, 14 Dec 2016 14:41:58 +0300 Subject: [PATCH 25/57] Request timeout for _create_bucket and _put_object --- inc/libs3.h | 4 + inc/request.h | 3 + src/acl.c | 6 +- src/bucket.c | 13 ++- src/multipart.c | 18 ++-- src/object.c | 16 ++- src/request.c | 6 ++ src/s3.c | 194 +++++++++++++++++------------------ src/service.c | 3 +- src/service_access_logging.c | 6 +- 10 files changed, 152 insertions(+), 117 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index 4e678cd..a4a4754 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -1837,6 +1837,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1848,6 +1849,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *authRegion, S3CannedAcl cannedAcl, const char *locationConstraint, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -1929,6 +1931,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1938,6 +1941,7 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, uint64_t contentLength, const S3PutProperties *putProperties, S3RequestContext *requestContext, + int timeoutMs, const S3PutObjectHandler *handler, void *callbackData); diff --git a/inc/request.h b/inc/request.h index 7d9cb03..041d548 100644 --- a/inc/request.h +++ b/inc/request.h @@ -102,6 +102,9 @@ typedef struct RequestParams // Data passed to the callbacks void *callbackData; + + // Request timeout. 0 if not set + int timeoutMs; } RequestParams; diff --git a/src/acl.c b/src/acl.c index c2eeb08..f748e7a 100644 --- a/src/acl.c +++ b/src/acl.c @@ -144,7 +144,8 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize &getAclDataCallback, // fromS3Callback &getAclCompleteCallback, // completeCallback - gaData // callbackData + gaData, // callbackData + 0 // timeoutMs }; // Perform the request @@ -344,7 +345,8 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, data->aclXmlDocumentLen, // toS3CallbackTotalSize 0, // fromS3Callback &setAclCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + 0 // timeoutMs }; // Perform the request diff --git a/src/bucket.c b/src/bucket.c index ab12c8f..3b54b1f 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -157,7 +157,8 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, 0, // toS3CallbackTotalSize &testBucketDataCallback, // fromS3Callback &testBucketCompleteCallback, // completeCallback - tbData // callbackData + tbData, // callbackData + 0 // timeoutMs }; // Perform the request @@ -231,6 +232,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *authRegion, S3CannedAcl cannedAcl, const char *locationConstraint, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data @@ -298,7 +300,8 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, cbData->docLen, // toS3CallbackTotalSize 0, // fromS3Callback &createBucketCompleteCallback, // completeCallback - cbData // callbackData + cbData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -384,7 +387,8 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, 0, // toS3CallbackTotalSize 0, // fromS3Callback &deleteBucketCompleteCallback, // completeCallback - dbData // callbackData + dbData, // callbackData + 0 // timeoutMs }; // Perform the request @@ -746,7 +750,8 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, 0, // toS3CallbackTotalSize &listBucketDataCallback, // fromS3Callback &listBucketCompleteCallback, // completeCallback - lbData // callbackData + lbData, // callbackData + 0 // timeoutMs }; // Perform the request diff --git a/src/multipart.c b/src/multipart.c index 3101d0b..912f92a 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -133,7 +133,8 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize InitialMultipartCallback, // fromS3Callback InitialMultipartCompleteCallback, // completeCallback - mdata // callbackData + mdata, // callbackData + 0 // timeoutMs }; // Perform the request @@ -173,7 +174,8 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize 0, // fromS3Callback AbortMultipartUploadCompleteCallback, // completeCallback - 0 // callbackData + 0, // callbackData + 0 // timeoutMs }; // Perform the request @@ -219,7 +221,8 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, partContentLength, // toS3CallbackTotalSize 0, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + 0 // timeoutMs }; request_perform(¶ms, requestContext); @@ -357,7 +360,8 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, contentLength, // toS3CallbackTotalSize commitMultipartCallback, // fromS3Callback commitMultipartCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + 0 // timeoutMs }; request_perform(¶ms, requestContext); @@ -950,7 +954,8 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, 0, // toS3CallbackTotalSize &listMultipartDataCallback, // fromS3Callback &listMultipartCompleteCallback, // completeCallback - lmData // callbackData + lmData, // callbackData + 0 // timeoutMs }; // Perform the request @@ -1072,7 +1077,8 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize &listPartsDataCallback, // fromS3Callback &listPartsCompleteCallback, // completeCallback - lpData // callbackData + lpData, // callbackData + 0 // timeoutMs }; // Perform the request diff --git a/src/object.c b/src/object.c index cc2be6e..6f9ca8d 100644 --- a/src/object.c +++ b/src/object.c @@ -36,6 +36,7 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, uint64_t contentLength, const S3PutProperties *putProperties, S3RequestContext *requestContext, + int timeoutMs, const S3PutObjectHandler *handler, void *callbackData) { // Set up the RequestParams @@ -64,7 +65,8 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, contentLength, // toS3CallbackTotalSize 0, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + timeoutMs // timeoutMs }; // Perform the request @@ -255,7 +257,8 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize ©ObjectDataCallback, // fromS3Callback ©ObjectCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + 0 // timeoutMs }; // Perform the request @@ -297,7 +300,8 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize handler->getObjectDataCallback, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + 0 // timeoutMs }; // Perform the request @@ -337,7 +341,8 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize 0, // fromS3Callback handler->completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + 0, // timeoutMs }; // Perform the request @@ -377,7 +382,8 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, 0, // toS3CallbackTotalSize 0, // fromS3Callback handler->completeCallback, // completeCallback - callbackData // callbackData + callbackData, // callbackData + 0 // timeoutMs }; // Perform the request diff --git a/src/request.c b/src/request.c index 97ed1dd..2d5c2ce 100644 --- a/src/request.c +++ b/src/request.c @@ -1217,6 +1217,12 @@ static S3Status setup_curl(Request *request, curl_easy_setopt_safe(CURLOPT_LOW_SPEED_LIMIT, 1024); curl_easy_setopt_safe(CURLOPT_LOW_SPEED_TIME, 15); + + if (params->timeoutMs > 0) { + curl_easy_setopt_safe(CURLOPT_TIMEOUT_MS, params->timeoutMs); + } + + // Append standard headers #define append_standard_header(fieldName) \ if (values-> fieldName [0]) { \ diff --git a/src/s3.c b/src/s3.c index 797a124..e0c0248 100644 --- a/src/s3.c +++ b/src/s3.c @@ -1,10 +1,10 @@ /** ************************************************************************** * s3.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -170,7 +170,7 @@ static void S3_init() if ((status = S3_initialize("s3", verifyPeerG|S3_INIT_ALL, hostname)) != S3StatusOK) { - fprintf(stderr, "Failed to initialize libs3: %s\n", + fprintf(stderr, "Failed to initialize libs3: %s\n", S3_get_status_name(status)); exit(-1); } @@ -389,7 +389,7 @@ static uint64_t convertInt(const char *str, const char *paramName) while (*str) { if (!isdigit(*str)) { - fprintf(stderr, "\nERROR: Nondigit in %s parameter: %c\n", + fprintf(stderr, "\nERROR: Nondigit in %s parameter: %c\n", paramName, *str); usageExit(stderr); } @@ -452,7 +452,7 @@ static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) } -static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, +static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, char *buffer) { *amtReturn = 0; @@ -466,7 +466,7 @@ static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, *amtReturn = (buf->size > amt) ? amt : buf->size; memcpy(buffer, &(buf->data[buf->start]), *amtReturn); - + buf->start += *amtReturn, buf->size -= *amtReturn; if (buf->size == 0) { @@ -573,7 +573,7 @@ static int64_t parseIso8601Time(const char *str) str++; } } - + if (checkString(str, "-dd:dd") || checkString(str, "+dd:dd")) { int sign = (*str++ == '-') ? -1 : 1; int hours = nextnum(); @@ -614,7 +614,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, return 0; \ } \ } while (0) - + #define COPY_STRING_MAXLEN(field, maxlen) \ do { \ SKIP_SPACE(1); \ @@ -634,7 +634,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, if (!*aclXml) { break; } - + // Skip Type lines and dash lines if (!strncmp(aclXml, "Type", sizeof("Type") - 1) || (*aclXml == '-')) { @@ -643,7 +643,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, } continue; } - + if (!strncmp(aclXml, "OwnerID", sizeof("OwnerID") - 1)) { aclXml += sizeof("OwnerID") - 1; COPY_STRING_MAXLEN(ownerId, S3_MAX_GRANTEE_USER_ID_SIZE); @@ -684,7 +684,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, grant->granteeType = S3GranteeTypeAllUsers; aclXml += (sizeof("All Users") - 1); } - else if (!strncmp(aclXml, "Log Delivery", + else if (!strncmp(aclXml, "Log Delivery", sizeof("Log Delivery") - 1)) { grant->granteeType = S3GranteeTypeLogDelivery; aclXml += (sizeof("Log Delivery") - 1); @@ -698,7 +698,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, } SKIP_SPACE(1); - + if (!strncmp(aclXml, "READ_ACP", sizeof("READ_ACP") - 1)) { grant->permission = S3PermissionReadACP; aclXml += (sizeof("READ_ACP") - 1); @@ -715,7 +715,7 @@ static int convert_simple_acl(char *aclXml, char *ownerId, grant->permission = S3PermissionWrite; aclXml += (sizeof("WRITE") - 1); } - else if (!strncmp(aclXml, "FULL_CONTROL", + else if (!strncmp(aclXml, "FULL_CONTROL", sizeof("FULL_CONTROL") - 1)) { grant->permission = S3PermissionFullControl; aclXml += (sizeof("FULL_CONTROL") - 1); @@ -772,7 +772,7 @@ static S3Status responsePropertiesCallback printf("%s: %s\n", name, properties-> field); \ } \ } while (0) - + print_nonnull("Content-Type", contentType); print_nonnull("Request-Id", requestId); print_nonnull("Request-Id-2", requestId2); @@ -834,8 +834,8 @@ static void responseCompleteCallback(S3Status status, "%s", " Extra Details:\n"); int i; for (i = 0; i < error->extraDetailsCount; i++) { - len += snprintf(&(errorDetailsG[len]), - sizeof(errorDetailsG) - len, " %s: %s\n", + len += snprintf(&(errorDetailsG[len]), + sizeof(errorDetailsG) - len, " %s: %s\n", error->extraDetails[i].name, error->extraDetails[i].value); } @@ -857,7 +857,7 @@ static void printListServiceHeader(int allDetails) printf("%-56s %-20s", " Bucket", " Created"); if (allDetails) { - printf(" %-64s %-12s", + printf(" %-64s %-12s", " Owner ID", "Display Name"); } @@ -872,7 +872,7 @@ static void printListServiceHeader(int allDetails) } -static S3Status listServiceCallback(const char *ownerId, +static S3Status listServiceCallback(const char *ownerId, const char *ownerDisplayName, const char *bucketName, int64_t creationDate, void *callbackData) @@ -895,7 +895,7 @@ static S3Status listServiceCallback(const char *ownerId, printf("%-56s %-20s", bucketName, timebuf); if (data->allDetails) { - printf(" %-64s %-12s", ownerId ? ownerId : "", + printf(" %-64s %-12s", ownerId ? ownerId : "", ownerDisplayName ? ownerDisplayName : ""); } printf("\n"); @@ -1064,7 +1064,7 @@ static void create_bucket(int argc, char **argv, int optindex) do { S3_create_bucket(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, bucketName, awsRegionG, cannedAcl, locationConstraint, - 0, &responseHandler, 0); + 0, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -1127,12 +1127,12 @@ typedef struct list_bucket_callback_data static void printListBucketHeader(int allDetails) { - printf("%-50s %-20s %-5s", - " Key", + printf("%-50s %-20s %-5s", + " Key", " Last Modified", "Size"); if (allDetails) { - printf(" %-34s %-64s %-12s", - " ETag", + printf(" %-34s %-64s %-12s", + " ETag", " Owner ID", "Display Name"); } @@ -1149,13 +1149,13 @@ static void printListBucketHeader(int allDetails) static S3Status listBucketCallback(int isTruncated, const char *nextMarker, - int contentsCount, + int contentsCount, const S3ListBucketContent *contents, int commonPrefixesCount, const char **commonPrefixes, void *callbackData) { - list_bucket_callback_data *data = + list_bucket_callback_data *data = (list_bucket_callback_data *) callbackData; data->isTruncated = isTruncated; @@ -1167,13 +1167,13 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, nextMarker = contents[contentsCount - 1].key; } if (nextMarker) { - snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", + snprintf(data->nextMarker, sizeof(data->nextMarker), "%s", nextMarker); } else { data->nextMarker[0] = 0; } - + if (contentsCount && !data->keyCount) { printListBucketHeader(data->allDetails); } @@ -1199,14 +1199,14 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, } else { time_t t = (time_t) content->lastModified; - strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", + strftime(timebuf, sizeof(timebuf), "%Y-%m-%dT%H:%M:%SZ", gmtime(&t)); char sizebuf[16]; if (content->size < 100000) { sprintf(sizebuf, "%5llu", (unsigned long long) content->size); } else if (content->size < (1024 * 1024)) { - sprintf(sizebuf, "%4lluK", + sprintf(sizebuf, "%4lluK", ((unsigned long long) content->size) / 1024ULL); } else if (content->size < (10 * 1024 * 1024)) { @@ -1215,8 +1215,8 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, sprintf(sizebuf, "%1.2fM", f); } else if (content->size < (1024 * 1024 * 1024)) { - sprintf(sizebuf, "%4lluM", - ((unsigned long long) content->size) / + sprintf(sizebuf, "%4lluM", + ((unsigned long long) content->size) / (1024ULL * 1024ULL)); } else { @@ -1227,9 +1227,9 @@ static S3Status listBucketCallback(int isTruncated, const char *nextMarker, printf("%-50s %s %s", content->key, timebuf, sizebuf); if (data->allDetails) { printf(" %-34s %-64s %-12s", - content->eTag, + content->eTag, content->ownerId ? content->ownerId : "", - content->ownerDisplayName ? + content->ownerDisplayName ? content->ownerDisplayName : ""); } printf("\n"); @@ -1333,7 +1333,7 @@ static void list(int argc, char **argv, int optindex) else if (!strncmp(param, ALL_DETAILS_PREFIX, ALL_DETAILS_PREFIX_LEN)) { const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]); - if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || + if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || !strcmp(ad, "yes") || !strcmp(ad, "YES") || !strcmp(ad, "1")) { allDetails = 1; @@ -1349,7 +1349,7 @@ static void list(int argc, char **argv, int optindex) } if (bucketName) { - list_bucket(bucketName, prefix, marker, delimiter, maxkeys, + list_bucket(bucketName, prefix, marker, delimiter, maxkeys, allDetails); } else { @@ -1413,8 +1413,8 @@ static void printListMultipartHeader(int allDetails) static void printListPartsHeader() { - printf("%-25s %-30s %-30s %-15s", - "LastModified", + printf("%-25s %-30s %-30s %-15s", + "LastModified", "PartNumber", "ETag", "SIZE"); printf("\n"); @@ -1428,13 +1428,13 @@ static void printListPartsHeader() static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker, const char *nextUploadIdMarker, - int uploadsCount, + int uploadsCount, const S3ListMultipartUpload *uploads, int commonPrefixesCount, const char **commonPrefixes, void *callbackData) { - list_multiparts_callback_data *data = + list_multiparts_callback_data *data = (list_multiparts_callback_data *) callbackData; data->isTruncated = isTruncated; @@ -1447,7 +1447,7 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker nextKeyMarker = uploads[uploadsCount - 1].key; }*/ if (nextKeyMarker) { - snprintf(data->nextKeyMarker, sizeof(data->nextKeyMarker), "%s", + snprintf(data->nextKeyMarker, sizeof(data->nextKeyMarker), "%s", nextKeyMarker); } else { @@ -1455,7 +1455,7 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker } if (nextUploadIdMarker) { - snprintf(data->nextUploadIdMarker, sizeof(data->nextUploadIdMarker), "%s", + snprintf(data->nextUploadIdMarker, sizeof(data->nextUploadIdMarker), "%s", nextUploadIdMarker); } else { @@ -1498,12 +1498,12 @@ static S3Status listMultipartCallback(int isTruncated, const char *nextKeyMarker printf("%-50s %s %-50s", upload->key, timebuf, upload->uploadId); if (data->allDetails) { printf(" %-34s %-64s %-12s %-64s %-12s", - upload->storageClass, + upload->storageClass, upload->ownerId ? upload->ownerId : "", - upload->ownerDisplayName ? + upload->ownerDisplayName ? upload->ownerDisplayName : "", upload->initiatorId ? upload->initiatorId : "", - upload->initiatorDisplayName ? + upload->initiatorDisplayName ? upload->initiatorDisplayName : ""); } printf("\n"); @@ -1532,7 +1532,7 @@ static S3Status listPartsCallback(int isTruncated, const S3ListPart *parts, void *callbackData) { - list_parts_callback_data *data = + list_parts_callback_data *data = (list_parts_callback_data *) callbackData; data->isTruncated = isTruncated; @@ -1548,7 +1548,7 @@ static S3Status listPartsCallback(int isTruncated, }*/ if (nextPartNumberMarker) { snprintf(data->nextPartNumberMarker, - sizeof(data->nextPartNumberMarker), "%s", + sizeof(data->nextPartNumberMarker), "%s", nextPartNumberMarker); } else { @@ -1556,7 +1556,7 @@ static S3Status listPartsCallback(int isTruncated, } if (initiatorId) { - snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", + snprintf(data->initiatorId, sizeof(data->initiatorId), "%s", initiatorId); } else { @@ -1565,7 +1565,7 @@ static S3Status listPartsCallback(int isTruncated, if (initiatorDisplayName) { snprintf(data->initiatorDisplayName, - sizeof(data->initiatorDisplayName), "%s", + sizeof(data->initiatorDisplayName), "%s", initiatorDisplayName); } else { @@ -1573,7 +1573,7 @@ static S3Status listPartsCallback(int isTruncated, } if (ownerId) { - snprintf(data->ownerId, sizeof(data->ownerId), "%s", + snprintf(data->ownerId, sizeof(data->ownerId), "%s", ownerId); } else { @@ -1581,7 +1581,7 @@ static S3Status listPartsCallback(int isTruncated, } if (ownerDisplayName) { - snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", + snprintf(data->ownerDisplayName, sizeof(data->ownerDisplayName), "%s", ownerDisplayName); } else { @@ -1589,13 +1589,13 @@ static S3Status listPartsCallback(int isTruncated, } if (storageClass) { - snprintf(data->storageClass, sizeof(data->storageClass), "%s", + snprintf(data->storageClass, sizeof(data->storageClass), "%s", storageClass); } else { data->storageClass[0] = 0; } - + if (partsCount && !data->partsCount && !data->noPrint) { printListPartsHeader(); } @@ -1637,7 +1637,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) const char *encodingtype = 0, *uploadidmarker = 0; int maxuploads = 0, allDetails = 0; while (optindex < argc) { - char *param = argv[optindex++]; + char *param = argv[optindex++]; if (!strncmp(param, PREFIX_PREFIX, PREFIX_PREFIX_LEN)) { prefix = &(param[PREFIX_PREFIX_LEN]); } @@ -1661,7 +1661,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) } else if (!strncmp(param, ALL_DETAILS_PREFIX, ALL_DETAILS_PREFIX_LEN)) { const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]); - if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || + if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || !strcmp(ad, "yes") || !strcmp(ad, "YES") || !strcmp(ad, "1")) { allDetails = 1; @@ -1767,7 +1767,7 @@ static void list_parts(int argc, char **argv, int optindex) const char *encodingtype = 0; int allDetails = 0, maxparts = 0; while (optindex < argc) { - char *param = argv[optindex++]; + char *param = argv[optindex++]; if (!strncmp(param, UPLOAD_ID_PREFIX, UPLOAD_ID_PREFIX_LEN)) { uploadid = &(param[UPLOAD_ID_PREFIX_LEN]); } @@ -1788,7 +1788,7 @@ static void list_parts(int argc, char **argv, int optindex) else if (!strncmp(param, ALL_DETAILS_PREFIX, ALL_DETAILS_PREFIX_LEN)) { const char *ad = &(param[ALL_DETAILS_PREFIX_LEN]); - if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || + if (!strcmp(ad, "true") || !strcmp(ad, "TRUE") || !strcmp(ad, "yes") || !strcmp(ad, "YES") || !strcmp(ad, "1")) { allDetails = 1; @@ -1922,7 +1922,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) list_multiparts_callback_data data; memset(&data, 0, sizeof(list_multiparts_callback_data)); - if (keymarker != 0) { + if (keymarker != 0) { snprintf(data.nextKeyMarker, sizeof(data.nextKeyMarker), "%s", keymarker); } @@ -2011,7 +2011,7 @@ typedef struct put_object_callback_data static int putObjectDataCallback(int bufferSize, char *buffer, void *callbackData) { - put_object_callback_data *data = + put_object_callback_data *data = (put_object_callback_data *) callbackData; int ret = 0; @@ -2033,7 +2033,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, if (data->contentLength && !data->noStatus) { // Avoid a weird bug in MingW, which won't print the second integer // value properly when it's in the same call, so print separately - printf("%llu bytes remaining ", + printf("%llu bytes remaining ", (unsigned long long) data->totalContentLength); printf("(%d%% complete) ...\n", (int) (((data->totalOriginalContentLength - @@ -2090,7 +2090,7 @@ static int multipartPutXmlCallback(int bufferSize, char *buffer, } -static int try_get_parts_info(const char *bucketName, const char *key, +static int try_get_parts_info(const char *bucketName, const char *key, UploadManager *manager) { S3BucketContext bucketContext = @@ -2184,7 +2184,7 @@ static void put_object(int argc, char **argv, int optindex, if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { filename = &(param[FILENAME_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_LENGTH_PREFIX, + else if (!strncmp(param, CONTENT_LENGTH_PREFIX, CONTENT_LENGTH_PREFIX_LEN)) { contentLength = convertInt(&(param[CONTENT_LENGTH_PREFIX_LEN]), "contentLength"); @@ -2194,27 +2194,27 @@ static void put_object(int argc, char **argv, int optindex, usageExit(stderr); } } - else if (!strncmp(param, CACHE_CONTROL_PREFIX, + else if (!strncmp(param, CACHE_CONTROL_PREFIX, CACHE_CONTROL_PREFIX_LEN)) { cacheControl = &(param[CACHE_CONTROL_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_TYPE_PREFIX, + else if (!strncmp(param, CONTENT_TYPE_PREFIX, CONTENT_TYPE_PREFIX_LEN)) { contentType = &(param[CONTENT_TYPE_PREFIX_LEN]); } else if (!strncmp(param, MD5_PREFIX, MD5_PREFIX_LEN)) { md5 = &(param[MD5_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, + else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, CONTENT_DISPOSITION_FILENAME_PREFIX_LEN)) { - contentDispositionFilename = + contentDispositionFilename = &(param[CONTENT_DISPOSITION_FILENAME_PREFIX_LEN]); } - else if (!strncmp(param, CONTENT_ENCODING_PREFIX, + else if (!strncmp(param, CONTENT_ENCODING_PREFIX, CONTENT_ENCODING_PREFIX_LEN)) { contentEncoding = &(param[CONTENT_ENCODING_PREFIX_LEN]); } - else if (!strncmp(param, UPLOAD_ID_PREFIX, + else if (!strncmp(param, UPLOAD_ID_PREFIX, UPLOAD_ID_PREFIX_LEN)) { uploadId = &(param[UPLOAD_ID_PREFIX_LEN]); } @@ -2229,7 +2229,7 @@ static void put_object(int argc, char **argv, int optindex, else if (!strncmp(param, X_AMZ_META_PREFIX, X_AMZ_META_PREFIX_LEN)) { if (metaPropertiesCount == S3_MAX_METADATA_COUNT) { fprintf(stderr, "\nERROR: Too many x-amz-meta- properties, " - "limit %lu: %s\n", + "limit %lu: %s\n", (unsigned long) S3_MAX_METADATA_COUNT, param); usageExit(stderr); } @@ -2249,7 +2249,7 @@ static void put_object(int argc, char **argv, int optindex, else if (!strncmp(param, USE_SERVER_SIDE_ENCRYPTION_PREFIX, USE_SERVER_SIDE_ENCRYPTION_PREFIX_LEN)) { const char *val = &(param[USE_SERVER_SIDE_ENCRYPTION_PREFIX_LEN]); - if (!strcmp(val, "true") || !strcmp(val, "TRUE") || + if (!strcmp(val, "true") || !strcmp(val, "TRUE") || !strcmp(val, "yes") || !strcmp(val, "YES") || !strcmp(val, "1")) { useServerSideEncryption = 1; @@ -2279,7 +2279,7 @@ static void put_object(int argc, char **argv, int optindex, } else if (!strncmp(param, NO_STATUS_PREFIX, NO_STATUS_PREFIX_LEN)) { const char *ns = &(param[NO_STATUS_PREFIX_LEN]); - if (!strcmp(ns, "true") || !strcmp(ns, "TRUE") || + if (!strcmp(ns, "true") || !strcmp(ns, "TRUE") || !strcmp(ns, "yes") || !strcmp(ns, "YES") || !strcmp(ns, "1")) { noStatus = 1; @@ -2350,7 +2350,7 @@ static void put_object(int argc, char **argv, int optindex, } data.totalContentLength = - data.totalOriginalContentLength = + data.totalOriginalContentLength = data.contentLength = data.originalContentLength = contentLength; @@ -2392,7 +2392,7 @@ static void put_object(int argc, char **argv, int optindex, do { S3_put_object(&bucketContext, key, contentLength, &putProperties, 0, - &putObjectHandler, &data); + 0, &putObjectHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (data.infile) { @@ -2546,7 +2546,7 @@ static void put_object(int argc, char **argv, int optindex, do { S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, - 0, &manager); + 0, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { printError(); @@ -2683,23 +2683,23 @@ static void copy_object(int argc, char **argv, int optindex) while (optindex < argc) { char *param = argv[optindex++]; - if (!strncmp(param, CACHE_CONTROL_PREFIX, + if (!strncmp(param, CACHE_CONTROL_PREFIX, CACHE_CONTROL_PREFIX_LEN)) { cacheControl = &(param[CACHE_CONTROL_PREFIX_LEN]); anyPropertiesSet = 1; } - else if (!strncmp(param, CONTENT_TYPE_PREFIX, + else if (!strncmp(param, CONTENT_TYPE_PREFIX, CONTENT_TYPE_PREFIX_LEN)) { contentType = &(param[CONTENT_TYPE_PREFIX_LEN]); anyPropertiesSet = 1; } - else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, + else if (!strncmp(param, CONTENT_DISPOSITION_FILENAME_PREFIX, CONTENT_DISPOSITION_FILENAME_PREFIX_LEN)) { - contentDispositionFilename = + contentDispositionFilename = &(param[CONTENT_DISPOSITION_FILENAME_PREFIX_LEN]); anyPropertiesSet = 1; } - else if (!strncmp(param, CONTENT_ENCODING_PREFIX, + else if (!strncmp(param, CONTENT_ENCODING_PREFIX, CONTENT_ENCODING_PREFIX_LEN)) { contentEncoding = &(param[CONTENT_ENCODING_PREFIX_LEN]); anyPropertiesSet = 1; @@ -2716,7 +2716,7 @@ static void copy_object(int argc, char **argv, int optindex) else if (!strncmp(param, X_AMZ_META_PREFIX, X_AMZ_META_PREFIX_LEN)) { if (metaPropertiesCount == S3_MAX_METADATA_COUNT) { fprintf(stderr, "\nERROR: Too many x-amz-meta- properties, " - "limit %lu: %s\n", + "limit %lu: %s\n", (unsigned long) S3_MAX_METADATA_COUNT, param); usageExit(stderr); } @@ -2736,7 +2736,7 @@ static void copy_object(int argc, char **argv, int optindex) } else if (!strncmp(param, USE_SERVER_SIDE_ENCRYPTION_PREFIX, USE_SERVER_SIDE_ENCRYPTION_PREFIX_LEN)) { - if (!strcmp(param, "true") || !strcmp(param, "TRUE") || + if (!strcmp(param, "true") || !strcmp(param, "TRUE") || !strcmp(param, "yes") || !strcmp(param, "YES") || !strcmp(param, "1")) { useServerSideEncryption = 1; @@ -2843,7 +2843,7 @@ static S3Status getObjectDataCallback(int bufferSize, const char *buffer, size_t wrote = fwrite(buffer, 1, bufferSize, outfile); - return ((wrote < (size_t) bufferSize) ? + return ((wrote < (size_t) bufferSize) ? S3StatusAbortedByCallback : S3StatusOK); } @@ -2880,7 +2880,7 @@ static void get_object(int argc, char **argv, int optindex) if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { filename = &(param[FILENAME_PREFIX_LEN]); } - else if (!strncmp(param, IF_MODIFIED_SINCE_PREFIX, + else if (!strncmp(param, IF_MODIFIED_SINCE_PREFIX, IF_MODIFIED_SINCE_PREFIX_LEN)) { // Parse ifModifiedSince ifModifiedSince = parseIso8601Time @@ -2891,7 +2891,7 @@ static void get_object(int argc, char **argv, int optindex) usageExit(stderr); } } - else if (!strncmp(param, IF_NOT_MODIFIED_SINCE_PREFIX, + else if (!strncmp(param, IF_NOT_MODIFIED_SINCE_PREFIX, IF_NOT_MODIFIED_SINCE_PREFIX_LEN)) { // Parse ifModifiedSince ifNotModifiedSince = parseIso8601Time @@ -3044,7 +3044,7 @@ static void head_object(int argc, char **argv, int optindex) }; S3ResponseHandler responseHandler = - { + { &responsePropertiesCallback, &responseCompleteCallback }; @@ -3239,13 +3239,13 @@ void get_acl(int argc, char **argv, int optindex) }; do { - S3_get_acl(&bucketContext, key, ownerId, ownerDisplayName, + S3_get_acl(&bucketContext, key, ownerId, ownerDisplayName, &aclGrantCount, aclGrants, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { fprintf(outfile, "OwnerID %s %s\n", ownerId, ownerDisplayName); - fprintf(outfile, "%-6s %-90s %-12s\n", " Type", + fprintf(outfile, "%-6s %-90s %-12s\n", " Type", " User Identifier", " Permission"); fprintf(outfile, "------ " @@ -3255,7 +3255,7 @@ void get_acl(int argc, char **argv, int optindex) for (i = 0; i < aclGrantCount; i++) { S3AclGrant *grant = &(aclGrants[i]); const char *type; - char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + + char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16]; const char *id; @@ -3499,7 +3499,7 @@ void get_logging(int argc, char **argv, int optindex) do { S3_get_server_access_logging(&bucketContext, targetBucket, targetPrefix, - &aclGrantCount, aclGrants, 0, + &aclGrantCount, aclGrants, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -3509,7 +3509,7 @@ void get_logging(int argc, char **argv, int optindex) if (targetPrefix[0]) { printf("Target Prefix: %s\n", targetPrefix); } - fprintf(outfile, "%-6s %-90s %-12s\n", " Type", + fprintf(outfile, "%-6s %-90s %-12s\n", " Type", " User Identifier", " Permission"); fprintf(outfile, "------ " @@ -3519,7 +3519,7 @@ void get_logging(int argc, char **argv, int optindex) for (i = 0; i < aclGrantCount; i++) { S3AclGrant *grant = &(aclGrants[i]); const char *type; - char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + + char composedId[S3_MAX_GRANTEE_USER_ID_SIZE + S3_MAX_GRANTEE_DISPLAY_NAME_SIZE + 16]; const char *id; @@ -3597,7 +3597,7 @@ void set_logging(int argc, char **argv, int optindex) if (!strncmp(param, TARGET_BUCKET_PREFIX, TARGET_BUCKET_PREFIX_LEN)) { targetBucket = &(param[TARGET_BUCKET_PREFIX_LEN]); } - else if (!strncmp(param, TARGET_PREFIX_PREFIX, + else if (!strncmp(param, TARGET_PREFIX_PREFIX, TARGET_PREFIX_PREFIX_LEN)) { targetPrefix = &(param[TARGET_PREFIX_PREFIX_LEN]); } @@ -3666,8 +3666,8 @@ void set_logging(int argc, char **argv, int optindex) }; do { - S3_set_server_access_logging(&bucketContext, targetBucket, - targetPrefix, aclGrantCount, aclGrants, + S3_set_server_access_logging(&bucketContext, targetBucket, + targetPrefix, aclGrantCount, aclGrants, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -3750,7 +3750,7 @@ int main(int argc, char **argv) } secretAccessKeyG = getenv("S3_SECRET_ACCESS_KEY"); if (!secretAccessKeyG) { - fprintf(stderr, + fprintf(stderr, "Missing environment variable: S3_SECRET_ACCESS_KEY\n"); return -1; } @@ -3766,7 +3766,7 @@ int main(int argc, char **argv) } else if (!strcmp(command, "delete")) { if (optind == argc) { - fprintf(stderr, + fprintf(stderr, "\nERROR: Missing parameter: bucket or bucket/key\n"); usageExit(stderr); } diff --git a/src/service.c b/src/service.c index b4b3123..dcf8383 100644 --- a/src/service.c +++ b/src/service.c @@ -184,7 +184,8 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, 0, // toS3CallbackTotalSize &dataCallback, // fromS3Callback &completeCallback, // completeCallback - data // callbackData + data, // callbackData + 0 // timeoutMs }; // Perform the request diff --git a/src/service_access_logging.c b/src/service_access_logging.c index dc90a03..6152d0e 100644 --- a/src/service_access_logging.c +++ b/src/service_access_logging.c @@ -345,7 +345,8 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, 0, // toS3CallbackTotalSize &getBlsDataCallback, // fromS3Callback &getBlsCompleteCallback, // completeCallback - gsData // callbackData + gsData, // callbackData + 0 // timeoutMs }; // Perform the request @@ -551,7 +552,8 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, data->salXmlDocumentLen, // toS3CallbackTotalSize 0, // fromS3Callback &setSalCompleteCallback, // completeCallback - data // callbackData + data, // callbackData + 0 // timeoutMs }; // Perform the request From c4acef824d079970036787c13b63048b079c0c7a Mon Sep 17 00:00:00 2001 From: "sergey.dobrodey" Date: Wed, 11 Jan 2017 14:21:01 +0300 Subject: [PATCH 26/57] Request timeout for _get_object and _head_object --- inc/libs3.h | 4 ++++ src/object.c | 6 ++++-- src/s3.c | 4 ++-- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index a4a4754..d3865a9 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -2055,6 +2055,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2064,6 +2065,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, const S3GetConditions *getConditions, uint64_t startByte, uint64_t byteCount, S3RequestContext *requestContext, + int timeoutMs, const S3GetObjectHandler *handler, void *callbackData); @@ -2076,6 +2078,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2083,6 +2086,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, **/ void S3_head_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); /** diff --git a/src/object.c b/src/object.c index 6f9ca8d..e4eb0cc 100644 --- a/src/object.c +++ b/src/object.c @@ -272,6 +272,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, const S3GetConditions *getConditions, uint64_t startByte, uint64_t byteCount, S3RequestContext *requestContext, + int timeoutMs, const S3GetObjectHandler *handler, void *callbackData) { // Set up the RequestParams @@ -301,7 +302,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, handler->getObjectDataCallback, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback callbackData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -313,6 +314,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, void S3_head_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Set up the RequestParams @@ -342,7 +344,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, 0, // fromS3Callback handler->completeCallback, // completeCallback callbackData, // callbackData - 0, // timeoutMs + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/s3.c b/src/s3.c index e0c0248..39d56f7 100644 --- a/src/s3.c +++ b/src/s3.c @@ -2983,7 +2983,7 @@ static void get_object(int argc, char **argv, int optindex) do { S3_get_object(&bucketContext, key, &getConditions, startByte, - byteCount, 0, &getObjectHandler, outfile); + byteCount, 0, 0, &getObjectHandler, outfile); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -3050,7 +3050,7 @@ static void head_object(int argc, char **argv, int optindex) }; do { - S3_head_object(&bucketContext, key, 0, &responseHandler, 0); + S3_head_object(&bucketContext, key, 0, 0, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if ((statusG != S3StatusOK) && From da639f3d2cbf0c9daad588949fe5db4e58235c79 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 3 Feb 2017 18:10:08 +0300 Subject: [PATCH 27/57] remove trailing whitespace from some files --- inc/libs3.h | 176 +++++++++++++++++----------------- src/acl.c | 32 +++---- src/bucket.c | 68 ++++++------- src/multipart.c | 178 +++++++++++++++++------------------ src/object.c | 18 ++-- src/request.c | 64 ++++++------- src/service.c | 18 ++-- src/service_access_logging.c | 56 +++++------ 8 files changed, 305 insertions(+), 305 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index d3865a9..5af6c07 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -2,9 +2,9 @@ * @file libs3.h * @details * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -95,10 +95,10 @@ extern "C" { * user of S3, it is always possible that these maximums may be too low in * some rare circumstances. Bug reports should this unlikely situation occur * would be most appreciated. - * + * * Threading Rules * --------------- - * + * * 1. All arguments passed to any function must not be modified directly until * the function returns. * 2. All S3RequestContext and S3Request arguments passed to all functions may @@ -207,7 +207,7 @@ extern "C" { /** * This constant is used by the S3_initialize() function, to specify that - * the winsock library should be initialized by libs3; only relevent on + * the winsock library should be initialized by libs3; only relevent on * Microsoft Windows platforms. **/ #define S3_INIT_WINSOCK 1 @@ -298,7 +298,7 @@ typedef enum S3StatusServerFailedVerification , S3StatusConnectionFailed , S3StatusAbortedByCallback , - + /** * Errors from the S3 service **/ @@ -484,7 +484,7 @@ typedef enum * Private canned ACL gives the owner FULL_CONTROL and no other permissions * are issued * Public Read canned ACL gives the owner FULL_CONTROL and all users Read - * permission + * permission * Public Read Write canned ACL gives the owner FULL_CONTROL and all users * Read and Write permission * AuthenticatedRead canned ACL gives the owner FULL_CONTROL and authenticated @@ -583,7 +583,7 @@ typedef struct S3ResponseProperties * modified time was not provided in the response. If this value is >= 0, * then the last modified date of the contents are available as a number * of seconds since the UNIX epoch. - * + * **/ int64_t lastModified; @@ -639,7 +639,7 @@ typedef struct S3AclGrant union { /** - * This structure is used iff the granteeType is + * This structure is used iff the granteeType is * S3GranteeTypeAmazonCustomerByEmail. **/ struct @@ -740,7 +740,7 @@ typedef struct S3ListBucketContent /** * This is the number of seconds since UNIX epoch of the last modified - * date of the object identified by the key. + * date of the object identified by the key. **/ int64_t lastModified; @@ -796,25 +796,25 @@ typedef struct S3ListMultipartUpload * access permissions allow it to be viewed. **/ const char *ownerDisplayName; - + const char *storageClass; /** * This is the number of seconds since UNIX epoch of the last modified - * date of the object identified by the key. + * date of the object identified by the key. **/ int64_t initiated; - + } S3ListMultipartUpload; typedef struct S3ListPart -{ +{ const char *eTag; /** * This is the number of seconds since UNIX epoch of the last modified - * date of the object identified by the key. + * date of the object identified by the key. **/ int64_t lastModified; uint64_t partNumber; @@ -1026,7 +1026,7 @@ typedef void (S3ResponseCompleteCallback)(S3Status status, const S3ErrorDetails *errorDetails, void *callbackData); - + /** * This callback is made for each bucket resulting from a list service * operation. @@ -1045,7 +1045,7 @@ typedef void (S3ResponseCompleteCallback)(S3Status status, * Typically, this will return either S3StatusOK or * S3StatusAbortedByCallback. **/ -typedef S3Status (S3ListServiceCallback)(const char *ownerId, +typedef S3Status (S3ListServiceCallback)(const char *ownerId, const char *ownerDisplayName, const char *bucketName, int64_t creationDateSeconds, @@ -1084,12 +1084,12 @@ typedef S3Status (S3ListServiceCallback)(const char *ownerId, **/ typedef S3Status (S3ListBucketCallback)(int isTruncated, const char *nextMarker, - int contentsCount, + int contentsCount, const S3ListBucketContent *contents, int commonPrefixesCount, const char **commonPrefixes, void *callbackData); - + /** * This callback is made during a put object operation, to obtain the next @@ -1133,7 +1133,7 @@ typedef int (S3PutObjectDataCallback)(int bufferSize, char *buffer, **/ typedef S3Status (S3GetObjectDataCallback)(int bufferSize, const char *buffer, void *callbackData); - + /** * This callback is made after initiation of a multipart upload operation. It @@ -1181,7 +1181,7 @@ typedef S3Status (S3MultipartInitialResponseCallback)(const char *upload_id, **/ typedef S3Status (S3ListMultipartUploadsResponseCallback) (int isTruncated, const char *nextKeyMarker, - const char *nextUploadIdMarker, int uploadsCount, + const char *nextUploadIdMarker, int uploadsCount, const S3ListMultipartUpload *uploads, int commonPrefixesCount, const char **commonPrefixes, void *callbackData); @@ -1254,7 +1254,7 @@ typedef struct S3ResponseHandler * if the response properties were not successfully returned from S3. **/ S3ResponsePropertiesCallback *propertiesCallback; - + /** * The completeCallback is always called for every request made to S3, * regardless of the outcome of the request. It provides the status of @@ -1380,7 +1380,7 @@ typedef struct S3ListMultipartUploadsHandler * responseHandler provides the properties and complete callback **/ S3ResponseHandler responseHandler; - + S3ListMultipartUploadsResponseCallback *responseXmlCallback; } S3ListMultipartUploadsHandler; @@ -1390,7 +1390,7 @@ typedef struct S3ListPartsHandler * responseHandler provides the properties and complete callback **/ S3ResponseHandler responseHandler; - + S3ListPartsResponseCallback *responseXmlCallback; } S3ListPartsHandler; @@ -1400,7 +1400,7 @@ typedef struct S3AbortMultipartUploadHandler * responseHandler provides the properties and complete callback **/ S3ResponseHandler responseHandler; - + } S3AbortMultipartUploadHandler; /** ************************************************************************** @@ -1540,7 +1540,7 @@ S3Status S3_validate_bucket_name(const char *bucketName, S3UriStyle uriStyle); **/ S3Status S3_convert_acl(char *aclXml, char *ownerId, char *ownerDisplayName, int *aclGrantCountReturn, S3AclGrant *aclGrants); - + /** * Returns nonzero if the status indicates that the request should be @@ -1633,7 +1633,7 @@ S3Status S3_runall_request_context(S3RequestContext *requestContext); * S3StatusOutOfMemory if requests could not be processed due to * an out of memory error **/ -S3Status S3_runonce_request_context(S3RequestContext *requestContext, +S3Status S3_runonce_request_context(S3RequestContext *requestContext, int *requestsRemainingReturn); @@ -1713,7 +1713,7 @@ void S3_set_request_context_verify_peer(S3RequestContext *requestContext, * of authenticated query string request. * * @param buffer is the output buffer for the authenticated query string. - * It must be at least S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in + * It must be at least S3_MAX_AUTHENTICATED_QUERY_STRING_SIZE bytes in * length. * @param bucketContext gives the bucket and associated parameters for the * request to generate. @@ -1751,7 +1751,7 @@ S3Status S3_generate_authenticated_query_string * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. @@ -1760,7 +1760,7 @@ S3Status S3_generate_authenticated_query_string * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1784,7 +1784,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. @@ -1803,7 +1803,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1825,7 +1825,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. @@ -1839,7 +1839,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, * performs the request immediately and synchronously. * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1863,7 +1863,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, * buckets * @param secretAccessKey gives the Amazon Secret Access Key for which to list * owned buckets - * @param securityToken gives the security token used to generate the Temporary + * @param securityToken gives the security token used to generate the Temporary * Security Credentials * @param hostName is the S3 host name to use; if NULL is passed in, the * default S3 host as provided to S3_initialize() will be used. @@ -1873,7 +1873,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1901,12 +1901,12 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_list_bucket(const S3BucketContext *bucketContext, - const char *prefix, const char *marker, + const char *prefix, const char *marker, const char *delimiter, int maxkeys, S3RequestContext *requestContext, const S3ListBucketHandler *handler, void *callbackData); @@ -1933,7 +1933,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, * performs the request immediately and synchronously. * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -1968,14 +1968,14 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, * @param eTagReturn is a buffer into which the resulting eTag of the copied * object will be written * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2057,7 +2057,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, * performs the request immediately and synchronously. * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2080,7 +2080,7 @@ void S3_get_object(const S3BucketContext *bucketContext, const char *key, * performs the request immediately and synchronously. * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2088,7 +2088,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, int timeoutMs, const S3ResponseHandler *handler, void *callbackData); - + /** * Deletes an object from S3. * @@ -2099,7 +2099,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2134,13 +2134,13 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ -void S3_get_acl(const S3BucketContext *bucketContext, const char *key, +void S3_get_acl(const S3BucketContext *bucketContext, const char *key, char *ownerId, char *ownerDisplayName, - int *aclGrantCountReturn, S3AclGrant *aclGrants, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); @@ -2166,13 +2166,13 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ -void S3_set_acl(const S3BucketContext *bucketContext, const char *key, +void S3_set_acl(const S3BucketContext *bucketContext, const char *key, const char *ownerId, const char *ownerDisplayName, - int aclGrantCount, const S3AclGrant *aclGrants, + int aclGrantCount, const S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); @@ -2213,19 +2213,19 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_get_server_access_logging(const S3BucketContext *bucketContext, char *targetBucketReturn, char *targetPrefixReturn, - int *aclGrantCountReturn, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); - + /** * Sets the service access logging settings for a bucket. The service access @@ -2252,23 +2252,23 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ void S3_set_server_access_logging(const S3BucketContext *bucketContext, - const char *targetBucket, - const char *targetPrefix, int aclGrantCount, - const S3AclGrant *aclGrants, + const char *targetBucket, + const char *targetPrefix, int aclGrantCount, + const S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData); - + /** - * This operation initiates a multipart upload and returns an upload ID. - * This upload ID is used to associate all the parts in the specific - * multipart upload. You specify this upload ID in each of your subsequent + * This operation initiates a multipart upload and returns an upload ID. + * This upload ID is used to associate all the parts in the specific + * multipart upload. You specify this upload ID in each of your subsequent * upload part requests * * @param bucketContext gives the bucket and associated parameters for this @@ -2278,7 +2278,7 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, * @param putProperties optionally provides additional properties to apply to * the object that is being put to * @param handler gives the callbacks to call as the request is processed and - * completed + * completed * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. @@ -2302,11 +2302,11 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, * @param putProperties optionally provides additional properties to apply to * the object that is being put to * @param handler gives the callbacks to call as the request is processed and - * completed - * @param seq is a part number uniquely identifies a part and also - * defines its position within the object being created. - * @param upload_id get from S3_initiate_multipart return - * @param partContentLength gives the size of the part, in bytes + * completed + * @param seq is a part number uniquely identifies a part and also + * defines its position within the object being created. + * @param upload_id get from S3_initiate_multipart return + * @param partContentLength gives the size of the part, in bytes * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. @@ -2321,7 +2321,7 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, /** - * This operation completes a multipart upload by assembling previously + * This operation completes a multipart upload by assembling previously * uploaded parts. * * @param bucketContext gives the bucket and associated parameters for this @@ -2330,15 +2330,15 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, * @param key is the source key * @param handler gives the callbacks to call as the request is processed and * completed - * @param upload_id get from S3_initiate_multipart return - * @param contentLength gives the total size of the commit message, in bytes + * @param upload_id get from S3_initiate_multipart return + * @param contentLength gives the total size of the commit message, in bytes * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ -void S3_complete_multipart_upload(S3BucketContext *bucketContext, +void S3_complete_multipart_upload(S3BucketContext *bucketContext, const char *key, S3MultipartCommitHandler *handler, const char *upload_id, @@ -2348,7 +2348,7 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, /** - * This operation lists the parts that have been uploaded for a specific + * This operation lists the parts that have been uploaded for a specific * multipart upload. * * @param bucketContext gives the bucket and associated parameters for this @@ -2359,10 +2359,10 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, * which listing should begin. Only parts with higher part numbers * will be listed. * @param uploadid identifying the multipart upload whose parts are being - * listed. + * listed. * @param encodingtype if present and non-empty, requests Amazon S3 to encode * the response and specifies the encoding method to use. - * @param maxparts Sets the maximum number of parts to return in the response + * @param maxparts Sets the maximum number of parts to return in the response * body. Default: 1,000 * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, @@ -2374,21 +2374,21 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, **/ void S3_list_parts(S3BucketContext *bucketContext, const char *key, const char *partnumbermarker, - const char *uploadid, const char *encodingtype, + const char *uploadid, const char *encodingtype, int maxparts, S3RequestContext *requestContext, const S3ListPartsHandler *handler, void *callbackData); /** - * This operation aborts a multipart upload. After a multipart upload is - * aborted, no additional parts can be uploaded using that upload ID. + * This operation aborts a multipart upload. After a multipart upload is + * aborted, no additional parts can be uploaded using that upload ID. * * @param bucketContext gives the bucket and associated parameters for this * request; this is the bucket for which service access logging is * being set * @param key is the source key * @param uploadId identifying the multipart upload whose parts are being - * listed. + * listed. * @param handler gives the callbacks to call as the request is processed and * completed **/ @@ -2398,9 +2398,9 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, /** - * This operation lists in-progress multipart uploads. An in-progress - * multipart upload is a multipart upload that has been initiated, - * using the Initiate Multipart Upload request, but has not yet been + * This operation lists in-progress multipart uploads. An in-progress + * multipart upload is a multipart upload that has been initiated, + * using the Initiate Multipart Upload request, but has not yet been * completed or aborted. * * @param bucketContext gives the bucket and associated parameters for this @@ -2416,9 +2416,9 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, * @param encodingtype if present and non-empty, requests Amazon S3 to encode * the response and specifies the encoding method to use. * @param delimiter if present and non-empty, is the character you use to - * group keys. - * @param maxuploads sets the maximum number of multipart uploads, - * from 1 to 1,000, to return in the response body. + * group keys. + * @param maxuploads sets the maximum number of multipart uploads, + * from 1 to 1,000, to return in the response body. * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. @@ -2426,11 +2426,11 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, * completed * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request - **/ -void S3_list_multipart_uploads(S3BucketContext *bucketContext, + **/ +void S3_list_multipart_uploads(S3BucketContext *bucketContext, const char *prefix, const char *keymarker, const char *uploadidmarker, - const char *encodingtype, const char *delimiter, + const char *encodingtype, const char *delimiter, int maxuploads, S3RequestContext *requestContext, const S3ListMultipartUploadsHandler *handler, void *callbackData); diff --git a/src/acl.c b/src/acl.c index f748e7a..2d4ae96 100644 --- a/src/acl.c +++ b/src/acl.c @@ -1,10 +1,10 @@ /** ************************************************************************** * acl.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -55,7 +55,7 @@ static S3Status getAclPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { GetAclData *gaData = (GetAclData *) callbackData; - + return (*(gaData->responsePropertiesCallback)) (responseProperties, gaData->callbackData); } @@ -69,12 +69,12 @@ static S3Status getAclDataCallback(int bufferSize, const char *buffer, int fit; string_buffer_append(gaData->aclXmlDocument, buffer, bufferSize, fit); - + return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; } -static void getAclCompleteCallback(S3Status requestStatus, +static void getAclCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -94,9 +94,9 @@ static void getAclCompleteCallback(S3Status requestStatus, } -void S3_get_acl(const S3BucketContext *bucketContext, const char *key, +void S3_get_acl(const S3BucketContext *bucketContext, const char *key, char *ownerId, char *ownerDisplayName, - int *aclGrantCountReturn, S3AclGrant *aclGrants, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData) { @@ -155,9 +155,9 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, // set acl ------------------------------------------------------------------- -static S3Status generateAclXmlDocument(const char *ownerId, +static S3Status generateAclXmlDocument(const char *ownerId, const char *ownerDisplayName, - int aclGrantCount, + int aclGrantCount, const S3AclGrant *aclGrants, int *xmlDocumentLenReturn, char *xmlDocument, @@ -192,7 +192,7 @@ static S3Status generateAclXmlDocument(const char *ownerId, break; case S3GranteeTypeCanonicalUser: append("CanonicalUser\">%s%s", - grant->grantee.canonicalUser.id, + grant->grantee.canonicalUser.id, grant->grantee.canonicalUser.displayName); break; default: { // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: @@ -243,7 +243,7 @@ static S3Status setAclPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { SetAclData *paData = (SetAclData *) callbackData; - + return (*(paData->responsePropertiesCallback)) (responseProperties, paData->callbackData); } @@ -253,11 +253,11 @@ static int setAclDataCallback(int bufferSize, char *buffer, void *callbackData) { SetAclData *paData = (SetAclData *) callbackData; - int remaining = (paData->aclXmlDocumentLen - + int remaining = (paData->aclXmlDocumentLen - paData->aclXmlDocumentBytesWritten); int toCopy = bufferSize > remaining ? remaining : bufferSize; - + if (!toCopy) { return 0; } @@ -271,7 +271,7 @@ static int setAclDataCallback(int bufferSize, char *buffer, void *callbackData) } -static void setAclCompleteCallback(S3Status requestStatus, +static void setAclCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -305,7 +305,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, // Convert aclGrants to XML document S3Status status = generateAclXmlDocument (ownerId, ownerDisplayName, aclGrantCount, aclGrants, - &(data->aclXmlDocumentLen), data->aclXmlDocument, + &(data->aclXmlDocumentLen), data->aclXmlDocument, sizeof(data->aclXmlDocument)); if (status != S3StatusOK) { free(data); diff --git a/src/bucket.c b/src/bucket.c index 3b54b1f..18d822e 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -1,10 +1,10 @@ /** ************************************************************************** * bucket.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -70,7 +70,7 @@ static S3Status testBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { TestBucketData *tbData = (TestBucketData *) callbackData; - + return (*(tbData->responsePropertiesCallback)) (responseProperties, tbData->callbackData); } @@ -85,15 +85,15 @@ static S3Status testBucketDataCallback(int bufferSize, const char *buffer, } -static void testBucketCompleteCallback(S3Status requestStatus, +static void testBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { TestBucketData *tbData = (TestBucketData *) callbackData; // Copy the location constraint into the return buffer - snprintf(tbData->locationConstraintReturn, - tbData->locationConstraintReturnSize, "%s", + snprintf(tbData->locationConstraintReturn, + tbData->locationConstraintReturnSize, "%s", tbData->locationConstraint); (*(tbData->responseCompleteCallback)) @@ -114,7 +114,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - TestBucketData *tbData = + TestBucketData *tbData = (TestBucketData *) malloc(sizeof(TestBucketData)); if (!tbData) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -176,20 +176,20 @@ typedef struct CreateBucketData char doc[1024]; int docLen, docBytesWritten; -} CreateBucketData; - +} CreateBucketData; + static S3Status createBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { CreateBucketData *cbData = (CreateBucketData *) callbackData; - + return (*(cbData->responsePropertiesCallback)) (responseProperties, cbData->callbackData); } -static int createBucketDataCallback(int bufferSize, char *buffer, +static int createBucketDataCallback(int bufferSize, char *buffer, void *callbackData) { CreateBucketData *cbData = (CreateBucketData *) callbackData; @@ -201,7 +201,7 @@ static int createBucketDataCallback(int bufferSize, char *buffer, int remaining = (cbData->docLen - cbData->docBytesWritten); int toCopy = bufferSize > remaining ? remaining : bufferSize; - + if (!toCopy) { return 0; } @@ -214,7 +214,7 @@ static int createBucketDataCallback(int bufferSize, char *buffer, } -static void createBucketCompleteCallback(S3Status requestStatus, +static void createBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -236,7 +236,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - CreateBucketData *cbData = + CreateBucketData *cbData = (CreateBucketData *) malloc(sizeof(CreateBucketData)); if (!cbData) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -308,7 +308,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, request_perform(¶ms, requestContext); } - + // delete bucket ------------------------------------------------------------- typedef struct DeleteBucketData @@ -329,7 +329,7 @@ static S3Status deleteBucketPropertiesCallback } -static void deleteBucketCompleteCallback(S3Status requestStatus, +static void deleteBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -471,7 +471,7 @@ static S3Status make_list_bucket_callback(ListBucketData *lbData) S3ListBucketContent *contentDest = &(contents[i]); ListBucketContents *contentSrc = &(lbData->contents[i]); contentDest->key = contentSrc->key; - contentDest->lastModified = + contentDest->lastModified = parseIso8601Time(contentSrc->lastModified); contentDest->eTag = contentSrc->eTag; contentDest->size = parseUnsignedInt(contentSrc->size); @@ -490,7 +490,7 @@ static S3Status make_list_bucket_callback(ListBucketData *lbData) return (*(lbData->listBucketCallback)) (isTruncated, lbData->nextMarker, - contentsCount, contents, commonPrefixesCount, + contentsCount, contents, commonPrefixesCount, (const char **) commonPrefixes, lbData->callbackData); } @@ -511,39 +511,39 @@ static S3Status listBucketXmlCallback(const char *elementPath, string_buffer_append(lbData->nextMarker, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/Key")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->key, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListBucketResult/Contents/LastModified")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->lastModified, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/ETag")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->eTag, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/Size")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->size, data, dataLen, fit); } else if (!strcmp(elementPath, "ListBucketResult/Contents/Owner/ID")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append(contents->ownerId, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListBucketResult/Contents/Owner/DisplayName")) { - ListBucketContents *contents = + ListBucketContents *contents = &(lbData->contents[lbData->contentsCount]); string_buffer_append (contents->ownerDisplayName, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListBucketResult/CommonPrefixes/Prefix")) { int which = lbData->commonPrefixesCount; size_t oldLen = lbData->commonPrefixLens[which]; @@ -607,22 +607,22 @@ static S3Status listBucketPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { ListBucketData *lbData = (ListBucketData *) callbackData; - + return (*(lbData->responsePropertiesCallback)) (responseProperties, lbData->callbackData); } -static S3Status listBucketDataCallback(int bufferSize, const char *buffer, +static S3Status listBucketDataCallback(int bufferSize, const char *buffer, void *callbackData) { ListBucketData *lbData = (ListBucketData *) callbackData; - + return simplexml_add(&(lbData->simpleXml), buffer, bufferSize); } -static void listBucketCompleteCallback(S3Status requestStatus, +static void listBucketCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -713,10 +713,10 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, simplexml_initialize(&(lbData->simpleXml), &listBucketXmlCallback, lbData); - lbData->responsePropertiesCallback = + lbData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lbData->listBucketCallback = handler->listBucketCallback; - lbData->responseCompleteCallback = + lbData->responseCompleteCallback = handler->responseHandler.completeCallback; lbData->callbackData = callbackData; diff --git a/src/multipart.c b/src/multipart.c index 912f92a..00f7480 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -1,10 +1,10 @@ /** ************************************************************************** * multipart.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -40,8 +40,8 @@ typedef struct InitialMultipartData void *userdata; } InitialMultipartData; -static S3Status InitialMultipartCallback(int bufferSize, const char *buffer, - void *callbackData) +static S3Status InitialMultipartCallback(int bufferSize, const char *buffer, + void *callbackData) { InitialMultipartData *mdata = (InitialMultipartData *) callbackData; return simplexml_add(&(mdata->simpleXml), buffer, bufferSize); @@ -49,7 +49,7 @@ static S3Status InitialMultipartCallback(int bufferSize, const char *buffer, static void InitialMultipartCompleteCallback (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, - void *callbackData) + void *callbackData) { InitialMultipartData *mdata = (InitialMultipartData *) callbackData; @@ -61,20 +61,20 @@ static void InitialMultipartCompleteCallback if (mdata->handler->responseXmlCallback) { (*mdata->handler->responseXmlCallback) (mdata->upload_id, mdata->userdata); - } + } simplexml_deinitialize(&(mdata->simpleXml)); free(mdata); -} +} static void AbortMultipartUploadCompleteCallback (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, - void *callbackData) -{ + void *callbackData) +{ (void) callbackData; (void) s3ErrorDetails; fprintf(stderr, "\nERROR: %s\n", S3_get_status_name(requestStatus)); - + } static S3Status initialMultipartXmlCallback(const char *elementPath, @@ -95,13 +95,13 @@ static S3Status initialMultipartXmlCallback(const char *elementPath, } void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, - S3PutProperties *putProperties, - S3MultipartInitialHandler *handler, - S3RequestContext *requestContext, - void *callbackData) + S3PutProperties *putProperties, + S3MultipartInitialHandler *handler, + S3RequestContext *requestContext, + void *callbackData) { - InitialMultipartData *mdata = - (InitialMultipartData *) malloc(sizeof(InitialMultipartData)); + InitialMultipartData *mdata = + (InitialMultipartData *) malloc(sizeof(InitialMultipartData)); simplexml_initialize(&(mdata->simpleXml), &initialMultipartXmlCallback, mdata); string_buffer_initialize(mdata->upload_id); @@ -144,7 +144,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, const char *uploadId, - S3AbortMultipartUploadHandler *handler) + S3AbortMultipartUploadHandler *handler) { char subResource[512]; snprintf(subResource, 512, "uploadId=%s", uploadId); @@ -230,7 +230,7 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, /* - * S3 commit multipart + * S3 commit multipart * */ @@ -238,15 +238,15 @@ typedef struct CommitMultiPartData { SimpleXml simplexml; void *userdata; S3MultipartCommitHandler *handler; - //response parsed from + //response parsed from string_buffer(location,128); string_buffer(etag,128); } CommitMultiPartData; -static S3Status commitMultipartResponseXMLcallback(const char *elementPath, - const char *data, - int dataLen, +static S3Status commitMultipartResponseXMLcallback(const char *elementPath, + const char *data, + int dataLen, void *callbackData) { int fit; @@ -260,13 +260,13 @@ static S3Status commitMultipartResponseXMLcallback(const char *elementPath, } } (void) fit; - + return S3StatusOK; } -static S3Status commitMultipartCallback(int bufferSize, const char *buffer, - void *callbackData) +static S3Status commitMultipartCallback(int bufferSize, const char *buffer, + void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData *) callbackData; return simplexml_add(&(data->simplexml), buffer, bufferSize); @@ -277,7 +277,7 @@ static S3Status commitMultipartPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData *) callbackData; - + if (data->handler->responseHandler.propertiesCallback) { (*(data->handler->responseHandler.propertiesCallback)) (responseProperties, data->userdata); @@ -286,8 +286,8 @@ static S3Status commitMultipartPropertiesCallback } static void commitMultipartCompleteCallback - (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, - void *callbackData) + (S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, + void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData*) callbackData; if (data->handler->responseHandler.completeCallback) { @@ -303,8 +303,8 @@ static void commitMultipartCompleteCallback } -static int commitMultipartPutObject(int bufferSize, char *buffer, - void *callbackData) +static int commitMultipartPutObject(int bufferSize, char *buffer, + void *callbackData) { CommitMultiPartData *data = (CommitMultiPartData*) callbackData; if (data->handler->putObjectDataCallback) { @@ -318,9 +318,9 @@ static int commitMultipartPutObject(int bufferSize, char *buffer, void S3_complete_multipart_upload(S3BucketContext *bucketContext, const char *key, - S3MultipartCommitHandler *handler, - const char *upload_id, int contentLength, - S3RequestContext *requestContext, + S3MultipartCommitHandler *handler, + const char *upload_id, int contentLength, + S3RequestContext *requestContext, void *callbackData) { char queryParams[512]; @@ -379,7 +379,7 @@ typedef struct ListMultipartUpload string_buffer(key, 1024); string_buffer(uploadId, 256); string_buffer(initiatorId, 256); - string_buffer(initiatorDisplayName, 256); + string_buffer(initiatorDisplayName, 256); string_buffer(ownerId, 256); string_buffer(ownerDisplayName, 256); string_buffer(storageClass, 256); @@ -392,7 +392,7 @@ typedef struct ListPart string_buffer(eTag, 1024); string_buffer(partNumber, 24); string_buffer(size, 256); - string_buffer(lastModified, 256); + string_buffer(lastModified, 256); } ListPart; @@ -429,12 +429,12 @@ typedef struct ListPartsData string_buffer(isTruncated, 64); string_buffer(nextPartNumberMarker, 1024); - string_buffer(initiatorId, 256); - string_buffer(initiatorDisplayName, 256); + string_buffer(initiatorId, 256); + string_buffer(initiatorDisplayName, 256); string_buffer(ownerId, 256); string_buffer(ownerDisplayName, 256); - string_buffer(storageClass, 256); - + string_buffer(storageClass, 256); + int handlePartsStart; int partsCount; ListPart parts[MAX_PARTS]; @@ -483,7 +483,7 @@ static S3Status listMultipartPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { ListMultipartData *lmData = (ListMultipartData *) callbackData; - + return (*(lmData->responsePropertiesCallback)) (responseProperties, lmData->callbackData); } @@ -493,26 +493,26 @@ static S3Status listPartsPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { ListPartsData *lpData = (ListPartsData *) callbackData; - + return (*(lpData->responsePropertiesCallback)) (responseProperties, lpData->callbackData); } -static S3Status listMultipartDataCallback(int bufferSize, const char *buffer, +static S3Status listMultipartDataCallback(int bufferSize, const char *buffer, void *callbackData) { ListMultipartData *lmData = (ListMultipartData *) callbackData; - + return simplexml_add(&(lmData->simpleXml), buffer, bufferSize); } -static S3Status listPartsDataCallback(int bufferSize, const char *buffer, +static S3Status listPartsDataCallback(int bufferSize, const char *buffer, void *callbackData) { ListPartsData *lpData = (ListPartsData *) callbackData; - + return simplexml_add(&(lpData->simpleXml), buffer, bufferSize); } @@ -573,11 +573,11 @@ static S3Status make_list_parts_callback(ListPartsData *lpData) S3ListPart *partDest = &(Parts[i]); ListPart *partSrc = &(lpData->parts[i]); partDest->eTag = partSrc->eTag; - partDest->partNumber = parseUnsignedInt(partSrc->partNumber); - partDest->size = parseUnsignedInt(partSrc->size); + partDest->partNumber = parseUnsignedInt(partSrc->partNumber); + partDest->size = parseUnsignedInt(partSrc->size); partDest->lastModified = parseIso8601Time(partSrc->lastModified); } - + return (*(lpData->listPartsCallback)) (isTruncated, lpData->nextPartNumberMarker, lpData->initiatorId, lpData->initiatorDisplayName, lpData->ownerId, @@ -586,7 +586,7 @@ static S3Status make_list_parts_callback(ListPartsData *lpData) } -static void listMultipartCompleteCallback(S3Status requestStatus, +static void listMultipartCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -606,7 +606,7 @@ static void listMultipartCompleteCallback(S3Status requestStatus, } -static void listPartsCompleteCallback(S3Status requestStatus, +static void listPartsCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -649,57 +649,57 @@ static S3Status listMultipartXmlCallback(const char *elementPath, } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Key")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->key, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Initiated")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->initiated, data, dataLen, fit); } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/UploadId")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->uploadId, data, dataLen, fit); } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Initiator/ID")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->initiatorId, data, dataLen, fit); } else if (!strcmp (elementPath, "ListMultipartUploadsResult/Upload/Initiator/DisplayName")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->initiatorDisplayName, data, dataLen, fit); } else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/Owner/ID")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->ownerId, data, dataLen, fit); } else if (!strcmp - (elementPath, + (elementPath, "ListMultipartUploadsResult/Upload/Owner/DisplayName")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append (uploads->ownerDisplayName, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListMultipartUploadsResult/Upload/StorageClass")) { - ListMultipartUpload *uploads = + ListMultipartUpload *uploads = &(lmData->uploads[lmData->uploadsCount]); string_buffer_append(uploads->storageClass, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListMultipartUploadsResult/CommonPrefixes/Prefix")) { int which = lmData->commonPrefixesCount; lmData->commonPrefixLens[which] += @@ -805,7 +805,7 @@ static S3Status listPartsXmlCallback(const char *elementPath, else if (!strcmp(elementPath, "ListPartsResult/Part/Size")) { ListPart *parts = &(lpData->parts[lpData->partsCount]); string_buffer_append(parts->size, data, dataLen, fit); - } + } } else { if (!strcmp(elementPath, "ListPartsResult/Part")) { @@ -824,7 +824,7 @@ static S3Status listPartsXmlCallback(const char *elementPath, // Initialize the next one initialize_list_part(&(lpData->parts[lpData->partsCount])); } - } + } } /* Avoid compiler error about variable set but not used */ @@ -837,7 +837,7 @@ static S3Status listPartsXmlCallback(const char *elementPath, void S3_list_multipart_uploads(S3BucketContext *bucketContext, const char *prefix, const char *keymarker, const char *uploadidmarker, - const char *encodingtype, const char *delimiter, + const char *encodingtype, const char *delimiter, int maxuploads, S3RequestContext *requestContext, const S3ListMultipartUploadsHandler *handler, void *callbackData) @@ -845,7 +845,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, // Compose the query params string_buffer(queryParams, 4096); string_buffer_initialize(queryParams); - + #define safe_append(name, value) \ do { \ int fit; \ @@ -879,8 +879,8 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, return; \ } \ } while (0) - - + + int amp = 0; if (prefix && *prefix) { safe_append("prefix", prefix); @@ -903,31 +903,31 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, maxuploads); safe_append("max-uploads", maxUploadsString); } - + ListMultipartData *lmData = (ListMultipartData *) malloc(sizeof(ListMultipartData)); - + if (!lmData) { (*(handler->responseHandler.completeCallback)) (S3StatusOutOfMemory, 0, callbackData); return; } - + simplexml_initialize(&(lmData->simpleXml), &listMultipartXmlCallback, lmData); - - lmData->responsePropertiesCallback = + + lmData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lmData->listMultipartCallback = handler->responseXmlCallback; - lmData->responseCompleteCallback = + lmData->responseCompleteCallback = handler->responseHandler.completeCallback; lmData->callbackData = callbackData; - + string_buffer_initialize(lmData->isTruncated); string_buffer_initialize(lmData->nextKeyMarker); string_buffer_initialize(lmData->nextUploadIdMarker); initialize_list_multipart_data(lmData); - + // Set up the RequestParams RequestParams params = { @@ -957,7 +957,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, lmData, // callbackData 0 // timeoutMs }; - + // Perform the request request_perform(¶ms, requestContext); } @@ -972,7 +972,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, // Compose the query params string_buffer(queryParams, 4096); string_buffer_initialize(queryParams); - + #define safe_append(name, value) \ do { \ int fit; \ @@ -1006,9 +1006,9 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, return; \ } \ } while (0) - + char subResource[512]; - snprintf(subResource, 512, "uploadId=%s", uploadid); + snprintf(subResource, 512, "uploadId=%s", uploadid); int amp = 0; if (partnumbermarker && *partnumbermarker) { @@ -1022,26 +1022,26 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, snprintf(maxPartsString, sizeof(maxPartsString), "%d", maxparts); safe_append("max-parts", maxPartsString); } - + ListPartsData *lpData = (ListPartsData *) malloc(sizeof(ListPartsData)); - + if (!lpData) { (*(handler->responseHandler.completeCallback)) (S3StatusOutOfMemory, 0, callbackData); return; } - + simplexml_initialize(&(lpData->simpleXml), &listPartsXmlCallback, lpData); - - lpData->responsePropertiesCallback = + + lpData->responsePropertiesCallback = handler->responseHandler.propertiesCallback; lpData->listPartsCallback = handler->responseXmlCallback; - lpData->responseCompleteCallback = + lpData->responseCompleteCallback = handler->responseHandler.completeCallback; lpData->callbackData = callbackData; - + string_buffer_initialize(lpData->isTruncated); string_buffer_initialize(lpData->nextPartNumberMarker); string_buffer_initialize(lpData->initiatorId); @@ -1080,7 +1080,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, lpData, // callbackData 0 // timeoutMs }; - + // Perform the request request_perform(¶ms, requestContext); } diff --git a/src/object.c b/src/object.c index e4eb0cc..cdfdbf6 100644 --- a/src/object.c +++ b/src/object.c @@ -1,10 +1,10 @@ /** ************************************************************************** * object.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -89,7 +89,7 @@ typedef struct CopyObjectData int eTagReturnSize; char *eTagReturn; int eTagReturnLen; - + string_buffer(lastModified, 256); } CopyObjectData; @@ -110,7 +110,7 @@ static S3Status copyObjectXmlCallback(const char *elementPath, if (coData->eTagReturnSize && coData->eTagReturn) { coData->eTagReturnLen += snprintf(&(coData->eTagReturn[coData->eTagReturnLen]), - coData->eTagReturnSize - + coData->eTagReturnSize - coData->eTagReturnLen - 1, "%.*s", dataLen, data); if (coData->eTagReturnLen >= coData->eTagReturnSize) { @@ -131,7 +131,7 @@ static S3Status copyObjectPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { CopyObjectData *coData = (CopyObjectData *) callbackData; - + return (*(coData->responsePropertiesCallback)) (responseProperties, coData->callbackData); } @@ -146,7 +146,7 @@ static S3Status copyObjectDataCallback(int bufferSize, const char *buffer, } -static void copyObjectCompleteCallback(S3Status requestStatus, +static void copyObjectCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -200,7 +200,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data - CopyObjectData *data = + CopyObjectData *data = (CopyObjectData *) malloc(sizeof(CopyObjectData)); if (!data) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); @@ -235,7 +235,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, { HttpRequestTypeCOPY, // httpRequestType { bucketContext->hostName, // hostName - destinationBucket ? destinationBucket : + destinationBucket ? destinationBucket : bucketContext->bucketName, // bucketName bucketContext->protocol, // protocol bucketContext->uriStyle, // uriStyle diff --git a/src/request.c b/src/request.c index 2d5c2ce..dc5bae8 100644 --- a/src/request.c +++ b/src/request.c @@ -1,10 +1,10 @@ /** ************************************************************************** * request.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -159,7 +159,7 @@ static void request_headers_done(Request *request) // Get the http response code long httpResponseCode; request->httpResponseCode = 0; - if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE, + if (curl_easy_getinfo(request->curl, CURLINFO_RESPONSE_CODE, &httpResponseCode) != CURLE_OK) { // Not able to get the HTTP response code - error request->status = S3StatusInternalError; @@ -169,7 +169,7 @@ static void request_headers_done(Request *request) request->httpResponseCode = httpResponseCode; } - response_headers_handler_done(&(request->responseHeadersHandler), + response_headers_handler_done(&(request->responseHeadersHandler), request->curl); // Only make the callback if it was a successful request; otherwise we're @@ -178,7 +178,7 @@ static void request_headers_done(Request *request) (request->httpResponseCode >= 200) && (request->httpResponseCode <= 299)) { request->status = (*(request->propertiesCallback)) - (&(request->responseHeadersHandler.responseProperties), + (&(request->responseHeadersHandler.responseProperties), request->callbackData); } } @@ -256,7 +256,7 @@ static size_t curl_write_func(void *ptr, size_t size, size_t nmemb, } // On HTTP error, we expect to parse an HTTP error response - if ((request->httpResponseCode < 200) || + if ((request->httpResponseCode < 200) || (request->httpResponseCode > 299)) { request->status = error_parser_add (&(request->errorParser), (char *) ptr, len); @@ -552,7 +552,7 @@ static S3Status compose_standard_headers(const RequestParams *params, S3StatusContentDispositionFilenameTooLong); // ContentEncoding - do_put_header("Content-Encoding: %s", contentEncoding, + do_put_header("Content-Encoding: %s", contentEncoding, contentEncodingHeader, S3StatusBadContentEncoding, S3StatusContentEncodingTooLong); @@ -599,21 +599,21 @@ static S3Status compose_standard_headers(const RequestParams *params, // If-None-Match header do_get_header("If-None-Match: %s", ifNotMatchETag, ifNoneMatchHeader, - S3StatusBadIfNotMatchETag, + S3StatusBadIfNotMatchETag, S3StatusIfNotMatchETagTooLong); // Range header if (params->startByte || params->byteCount) { if (params->byteCount) { snprintf(values->rangeHeader, sizeof(values->rangeHeader), - "Range: bytes=%llu-%llu", + "Range: bytes=%llu-%llu", (unsigned long long) params->startByte, - (unsigned long long) (params->startByte + + (unsigned long long) (params->startByte + params->byteCount - 1)); } else { snprintf(values->rangeHeader, sizeof(values->rangeHeader), - "Range: bytes=%llu-", + "Range: bytes=%llu-", (unsigned long long) params->startByte); } } @@ -720,7 +720,7 @@ static void canonicalize_signature_headers(RequestComputedValues *values) const char *c = header; char v; // If the header names are the same, append the next value - if ((i > 0) && + if ((i > 0) && !strncmp(header, sortedHeaders[i - 1], lastHeaderLen)) { // Replacing the previous newline with a comma *(buffer - 1) = ','; @@ -1156,18 +1156,18 @@ static S3Status setup_curl(Request *request, // Debugging only // curl_easy_setopt_safe(CURLOPT_VERBOSE, 1); - + // Set private data to request for the benefit of S3RequestContext curl_easy_setopt_safe(CURLOPT_PRIVATE, request); - + // Set header callback and data curl_easy_setopt_safe(CURLOPT_HEADERDATA, request); curl_easy_setopt_safe(CURLOPT_HEADERFUNCTION, &curl_header_func); - + // Set read callback, data, and readSize curl_easy_setopt_safe(CURLOPT_READFUNCTION, &curl_read_func); curl_easy_setopt_safe(CURLOPT_READDATA, request); - + // Set write callback and data curl_easy_setopt_safe(CURLOPT_WRITEFUNCTION, &curl_write_func); curl_easy_setopt_safe(CURLOPT_WRITEDATA, request); @@ -1193,7 +1193,7 @@ static S3Status setup_curl(Request *request, // I think this is useful - we don't need interactive performance, we need // to complete large operations quickly curl_easy_setopt_safe(CURLOPT_TCP_NODELAY, 1); - + // Don't use Curl's 'netrc' feature curl_easy_setopt_safe(CURLOPT_NETRC, CURL_NETRC_IGNORED); @@ -1237,14 +1237,14 @@ static S3Status setup_curl(Request *request, snprintf(header, sizeof(header), "Content-Length: %llu", (unsigned long long) params->toS3CallbackTotalSize); request->headers = curl_slist_append(request->headers, header); - request->headers = curl_slist_append(request->headers, + request->headers = curl_slist_append(request->headers, "Transfer-Encoding:"); } else if (params->httpRequestType == HttpRequestTypeCOPY) { - request->headers = curl_slist_append(request->headers, + request->headers = curl_slist_append(request->headers, "Transfer-Encoding:"); } - + append_standard_header(hostHeader); append_standard_header(cacheControlHeader); append_standard_header(contentTypeHeader); @@ -1262,7 +1262,7 @@ static S3Status setup_curl(Request *request, // Append x-amz- headers int i; for (i = 0; i < values->amzHeadersCount; i++) { - request->headers = + request->headers = curl_slist_append(request->headers, values->amzHeaders[i]); } @@ -1292,7 +1292,7 @@ static S3Status setup_curl(Request *request, default: // HttpRequestTypeGET break; } - + return S3StatusOK; } @@ -1302,7 +1302,7 @@ static void request_deinitialize(Request *request) if (request->headers) { curl_slist_free_all(request->headers); } - + error_parser_deinitialize(&(request->errorParser)); // curl_easy_reset prevents connections from being re-used for some @@ -1359,7 +1359,7 @@ static S3Status request_get(const RequestParams *params, // Compute the URL if ((status = compose_uri - (request->uri, sizeof(request->uri), + (request->uri, sizeof(request->uri), &(params->bucketContext), values->urlEncodedKey, params->subResource, params->queryParams)) != S3StatusOK) { curl_easy_cleanup(request->curl); @@ -1429,7 +1429,7 @@ static void request_release(Request *request) S3Status request_api_initialize(const char *userAgentInfo, int flags, const char *defaultHostName) { - if (curl_global_init(CURL_GLOBAL_ALL & + if (curl_global_init(CURL_GLOBAL_ALL & ~((flags & S3_INIT_WINSOCK) ? 0 : CURL_GLOBAL_WIN32)) != CURLE_OK) { return S3StatusInternalError; @@ -1440,7 +1440,7 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, defaultHostName = S3_DEFAULT_HOSTNAME; } - if (snprintf(defaultHostNameG, S3_MAX_HOSTNAME_SIZE, + if (snprintf(defaultHostNameG, S3_MAX_HOSTNAME_SIZE, "%s", defaultHostName) >= S3_MAX_HOSTNAME_SIZE) { return S3StatusUriTooLong; } @@ -1459,11 +1459,11 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, snprintf(platform, sizeof(platform), "Unknown"); } else { - snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname, + snprintf(platform, sizeof(platform), "%s%s%s", utsn.sysname, utsn.machine[0] ? " " : "", utsn.machine); } - snprintf(userAgentG, sizeof(userAgentG), + snprintf(userAgentG, sizeof(userAgentG), "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)", userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform); @@ -1617,11 +1617,11 @@ void request_finish(Request *request) // If we haven't detected this already, we now know that the headers are // definitely done being read in request_headers_done(request); - + // If there was no error processing the request, then possibly there was // an S3 error parsed, which should be converted into the request status if (request->status == S3StatusOK) { - error_parser_convert_status(&(request->errorParser), + error_parser_convert_status(&(request->errorParser), &(request->status)); // If there still was no error recorded, then it is possible that // there was in fact an error but that there was no error XML @@ -1647,7 +1647,7 @@ void request_finish(Request *request) case 400: request->status = S3StatusHttpErrorBadRequest; break; - case 403: + case 403: request->status = S3StatusHttpErrorForbidden; break; case 404: diff --git a/src/service.c b/src/service.c index dcf8383..2414513 100644 --- a/src/service.c +++ b/src/service.c @@ -1,10 +1,10 @@ /** ************************************************************************** * service.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -34,7 +34,7 @@ typedef struct XmlCallbackData { SimpleXml simpleXml; - + S3ResponsePropertiesCallback *responsePropertiesCallback; S3ListServiceCallback *listServiceCallback; S3ResponseCompleteCallback *responseCompleteCallback; @@ -58,16 +58,16 @@ static S3Status xmlCallback(const char *elementPath, const char *data, if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/ID")) { string_buffer_append(cbData->ownerId, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListAllMyBucketsResult/Owner/DisplayName")) { string_buffer_append(cbData->ownerDisplayName, data, dataLen, fit); } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "ListAllMyBucketsResult/Buckets/Bucket/Name")) { string_buffer_append(cbData->bucketName, data, dataLen, fit); } else if (!strcmp - (elementPath, + (elementPath, "ListAllMyBucketsResult/Buckets/Bucket/CreationDate")) { string_buffer_append(cbData->creationDate, data, dataLen, fit); } @@ -137,7 +137,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, const S3ListServiceHandler *handler, void *callbackData) { // Create and set up the callback data - XmlCallbackData *data = + XmlCallbackData *data = (XmlCallbackData *) malloc(sizeof(XmlCallbackData)); if (!data) { (*(handler->responseHandler.completeCallback)) @@ -157,7 +157,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, string_buffer_initialize(data->ownerDisplayName); string_buffer_initialize(data->bucketName); string_buffer_initialize(data->creationDate); - + // Set up the RequestParams RequestParams params = { diff --git a/src/service_access_logging.c b/src/service_access_logging.c index 6152d0e..49919c3 100644 --- a/src/service_access_logging.c +++ b/src/service_access_logging.c @@ -1,10 +1,10 @@ /** ************************************************************************** * server_access_logging.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -60,10 +60,10 @@ static S3Status convertBlsXmlCallback(const char *elementPath, if (data) { if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" "TargetBucket")) { - caData->targetBucketReturnLen += + caData->targetBucketReturnLen += snprintf(&(caData->targetBucketReturn [caData->targetBucketReturnLen]), - 255 - caData->targetBucketReturnLen - 1, + 255 - caData->targetBucketReturnLen - 1, "%.*s", dataLen, data); if (caData->targetBucketReturnLen >= 255) { return S3StatusTargetBucketTooLong; @@ -71,10 +71,10 @@ static S3Status convertBlsXmlCallback(const char *elementPath, } else if (!strcmp(elementPath, "BucketLoggingStatus/LoggingEnabled/" "TargetPrefix")) { - caData->targetPrefixReturnLen += + caData->targetPrefixReturnLen += snprintf(&(caData->targetPrefixReturn [caData->targetPrefixReturnLen]), - 255 - caData->targetPrefixReturnLen - 1, + 255 - caData->targetPrefixReturnLen - 1, "%.*s", dataLen, data); if (caData->targetPrefixReturnLen >= 255) { return S3StatusTargetPrefixTooLong; @@ -142,7 +142,7 @@ static S3Status convertBlsXmlCallback(const char *elementPath, else if (caData->userId[0] && caData->userDisplayName[0]) { grant->granteeType = S3GranteeTypeCanonicalUser; strcpy(grant->grantee.canonicalUser.id, caData->userId); - strcpy(grant->grantee.canonicalUser.displayName, + strcpy(grant->grantee.canonicalUser.displayName, caData->userDisplayName); } else if (caData->groupUri[0]) { @@ -223,7 +223,7 @@ static S3Status convert_bls(char *blsXml, char *targetBucketReturn, S3Status status = simplexml_add(&simpleXml, blsXml, strlen(blsXml)); simplexml_deinitialize(&simpleXml); - + return status; } @@ -252,7 +252,7 @@ static S3Status getBlsPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { GetBlsData *gsData = (GetBlsData *) callbackData; - + return (*(gsData->responsePropertiesCallback)) (responseProperties, gsData->callbackData); } @@ -266,12 +266,12 @@ static S3Status getBlsDataCallback(int bufferSize, const char *buffer, int fit; string_buffer_append(gsData->blsXmlDocument, buffer, bufferSize, fit); - + return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; } -static void getBlsCompleteCallback(S3Status requestStatus, +static void getBlsCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -281,7 +281,7 @@ static void getBlsCompleteCallback(S3Status requestStatus, // Parse the document requestStatus = convert_bls (gsData->blsXmlDocument, gsData->targetBucketReturn, - gsData->targetPrefixReturn, gsData->aclGrantCountReturn, + gsData->targetPrefixReturn, gsData->aclGrantCountReturn, gsData->aclGrants); } @@ -295,7 +295,7 @@ static void getBlsCompleteCallback(S3Status requestStatus, void S3_get_server_access_logging(const S3BucketContext *bucketContext, char *targetBucketReturn, char *targetPrefixReturn, - int *aclGrantCountReturn, + int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, @@ -359,7 +359,7 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, static S3Status generateSalXmlDocument(const char *targetBucket, const char *targetPrefix, - int aclGrantCount, + int aclGrantCount, const S3AclGrant *aclGrants, int *xmlDocumentLenReturn, char *xmlDocument, @@ -383,7 +383,7 @@ static S3Status generateSalXmlDocument(const char *targetBucket, if (targetBucket && targetBucket[0]) { append("%s", targetBucket); - append("%s", + append("%s", targetPrefix ? targetPrefix : ""); if (aclGrantCount) { @@ -403,7 +403,7 @@ static S3Status generateSalXmlDocument(const char *targetBucket, case S3GranteeTypeCanonicalUser: append("CanonicalUser\">%s%s" "", - grant->grantee.canonicalUser.id, + grant->grantee.canonicalUser.id, grant->grantee.canonicalUser.displayName); break; default: // case S3GranteeTypeAllAwsUsers/S3GranteeTypeAllUsers: @@ -415,9 +415,9 @@ static S3Status generateSalXmlDocument(const char *targetBucket, append("%s", ((grant->permission == S3PermissionRead) ? "READ" : (grant->permission == S3PermissionWrite) ? "WRITE" : - (grant->permission == + (grant->permission == S3PermissionReadACP) ? "READ_ACP" : - (grant->permission == + (grant->permission == S3PermissionWriteACP) ? "WRITE_ACP" : "FULL_CONTROL")); } append("%s", ""); @@ -448,7 +448,7 @@ static S3Status setSalPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { SetSalData *paData = (SetSalData *) callbackData; - + return (*(paData->responsePropertiesCallback)) (responseProperties, paData->callbackData); } @@ -458,11 +458,11 @@ static int setSalDataCallback(int bufferSize, char *buffer, void *callbackData) { SetSalData *paData = (SetSalData *) callbackData; - int remaining = (paData->salXmlDocumentLen - + int remaining = (paData->salXmlDocumentLen - paData->salXmlDocumentBytesWritten); int toCopy = bufferSize > remaining ? remaining : bufferSize; - + if (!toCopy) { return 0; } @@ -476,7 +476,7 @@ static int setSalDataCallback(int bufferSize, char *buffer, void *callbackData) } -static void setSalCompleteCallback(S3Status requestStatus, +static void setSalCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { @@ -490,9 +490,9 @@ static void setSalCompleteCallback(S3Status requestStatus, void S3_set_server_access_logging(const S3BucketContext *bucketContext, - const char *targetBucket, - const char *targetPrefix, int aclGrantCount, - const S3AclGrant *aclGrants, + const char *targetBucket, + const char *targetPrefix, int aclGrantCount, + const S3AclGrant *aclGrants, S3RequestContext *requestContext, const S3ResponseHandler *handler, void *callbackData) @@ -508,11 +508,11 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); return; } - + // Convert aclGrants to XML document S3Status status = generateSalXmlDocument (targetBucket, targetPrefix, aclGrantCount, aclGrants, - &(data->salXmlDocumentLen), data->salXmlDocument, + &(data->salXmlDocumentLen), data->salXmlDocument, sizeof(data->salXmlDocument)); if (status != S3StatusOK) { free(data); From a168d1fc0e98e92ed4509858b32e461b2671e1d2 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 3 Feb 2017 18:11:25 +0300 Subject: [PATCH 28/57] timeout: report timeout error if request timed out --- src/request.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/request.c b/src/request.c index dc5bae8..be431c0 100644 --- a/src/request.c +++ b/src/request.c @@ -1704,7 +1704,7 @@ S3Status request_curl_code_to_status(CURLcode code) return S3StatusFailedToConnect; case CURLE_WRITE_ERROR: case CURLE_OPERATION_TIMEDOUT: - return S3StatusConnectionFailed; + return S3StatusErrorRequestTimeout; case CURLE_PARTIAL_FILE: return S3StatusOK; #if LIBCURL_VERSION_NUM >= 0x071101 /* 7.17.1 */ From 552a45a9bcdfe18b139cfabd428694bca4ec756e Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 3 Feb 2017 18:11:57 +0300 Subject: [PATCH 29/57] set request timeout for all functions --- inc/libs3.h | 37 ++++++++++++++++++++++- src/acl.c | 6 ++-- src/bucket.c | 9 ++++-- src/multipart.c | 21 ++++++++----- src/object.c | 8 +++-- src/request.c | 2 +- src/s3.c | 57 ++++++++++++++++++++++++++---------- src/service.c | 3 +- src/service_access_logging.c | 6 ++-- 9 files changed, 114 insertions(+), 35 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index 5af6c07..fcfe224 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -1759,6 +1759,7 @@ S3Status S3_generate_authenticated_query_string * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1768,6 +1769,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *authRegion, S3RequestContext *requestContext, + int timeoutMs, const S3ListServiceHandler *handler, void *callbackData); @@ -1802,6 +1804,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1814,6 +1817,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, int locationConstraintReturnSize, char *locationConstraintReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -1872,6 +1876,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1882,6 +1887,7 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *securityToken, const char *hostName, const char *bucketName, const char *authRegion, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -1900,6 +1906,7 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1909,6 +1916,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, const char *marker, const char *delimiter, int maxkeys, S3RequestContext *requestContext, + int timeoutMs, const S3ListBucketHandler *handler, void *callbackData); @@ -1974,6 +1982,7 @@ void S3_put_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -1985,6 +1994,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2023,6 +2033,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2036,6 +2047,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2098,6 +2110,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2105,6 +2118,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, **/ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2133,6 +2147,7 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2142,6 +2157,7 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, char *ownerId, char *ownerDisplayName, int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2165,6 +2181,7 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2174,6 +2191,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, const char *ownerId, const char *ownerDisplayName, int aclGrantCount, const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2212,6 +2230,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2223,6 +2242,7 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2251,6 +2271,7 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2261,6 +2282,7 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, const char *targetPrefix, int aclGrantCount, const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData); @@ -2282,6 +2304,7 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2289,6 +2312,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, S3PutProperties *putProperties, S3MultipartInitialHandler *handler, S3RequestContext *requestContext, + int timeoutMs, void *callbackData); @@ -2310,6 +2334,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2317,7 +2342,9 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, S3PutProperties * putProperties, S3PutObjectHandler *handler, int seq, const char *upload_id, int partContentLength, - S3RequestContext *requestContext, void *callbackData); + S3RequestContext *requestContext, + int timeoutMs, + void *callbackData); /** @@ -2335,6 +2362,7 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param callbackData will be passed in as the callbackData parameter to * all callbacks for this request **/ @@ -2344,6 +2372,7 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, const char *upload_id, int contentLength, S3RequestContext *requestContext, + int timeoutMs, void *callbackData); @@ -2367,6 +2396,7 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2376,6 +2406,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, const char *partnumbermarker, const char *uploadid, const char *encodingtype, int maxparts, S3RequestContext *requestContext, + int timeoutMs, const S3ListPartsHandler *handler, void *callbackData); @@ -2389,11 +2420,13 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, * @param key is the source key * @param uploadId identifying the multipart upload whose parts are being * listed. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed **/ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, const char *uploadId, + int timeoutMs, S3AbortMultipartUploadHandler *handler); @@ -2422,6 +2455,7 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, * @param requestContext if non-NULL, gives the S3RequestContext to add this * request to, and does not perform the request immediately. If NULL, * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds * @param handler gives the callbacks to call as the request is processed and * completed * @param callbackData will be passed in as the callbackData parameter to @@ -2432,6 +2466,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, const char *uploadidmarker, const char *encodingtype, const char *delimiter, int maxuploads, S3RequestContext *requestContext, + int timeoutMs, const S3ListMultipartUploadsHandler *handler, void *callbackData); diff --git a/src/acl.c b/src/acl.c index 2d4ae96..04d977c 100644 --- a/src/acl.c +++ b/src/acl.c @@ -98,6 +98,7 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, char *ownerId, char *ownerDisplayName, int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data @@ -145,7 +146,7 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, &getAclDataCallback, // fromS3Callback &getAclCompleteCallback, // completeCallback gaData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -288,6 +289,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, const char *ownerId, const char *ownerDisplayName, int aclGrantCount, const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) { @@ -346,7 +348,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, 0, // fromS3Callback &setAclCompleteCallback, // completeCallback data, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/bucket.c b/src/bucket.c index 18d822e..0f852bc 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -111,6 +111,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, int locationConstraintReturnSize, char *locationConstraintReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data @@ -158,7 +159,7 @@ void S3_test_bucket(S3Protocol protocol, S3UriStyle uriStyle, &testBucketDataCallback, // fromS3Callback &testBucketCompleteCallback, // completeCallback tbData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -347,6 +348,7 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, const char *securityToken, const char *hostName, const char *bucketName, const char *authRegion, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data @@ -388,7 +390,7 @@ void S3_delete_bucket(S3Protocol protocol, S3UriStyle uriStyle, 0, // fromS3Callback &deleteBucketCompleteCallback, // completeCallback dbData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -645,6 +647,7 @@ static void listBucketCompleteCallback(S3Status requestStatus, void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, const char *marker, const char *delimiter, int maxkeys, S3RequestContext *requestContext, + int timeoutMs, const S3ListBucketHandler *handler, void *callbackData) { // Compose the query params @@ -751,7 +754,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, &listBucketDataCallback, // fromS3Callback &listBucketCompleteCallback, // completeCallback lbData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/multipart.c b/src/multipart.c index 00f7480..8b2fd32 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -98,6 +98,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, S3PutProperties *putProperties, S3MultipartInitialHandler *handler, S3RequestContext *requestContext, + int timeoutMs, void *callbackData) { InitialMultipartData *mdata = @@ -134,7 +135,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, InitialMultipartCallback, // fromS3Callback InitialMultipartCompleteCallback, // completeCallback mdata, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -144,6 +145,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, const char *uploadId, + int timeoutMs, S3AbortMultipartUploadHandler *handler) { char subResource[512]; @@ -175,7 +177,7 @@ void S3_abort_multipart_upload(S3BucketContext *bucketContext, const char *key, 0, // fromS3Callback AbortMultipartUploadCompleteCallback, // completeCallback 0, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -191,7 +193,9 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, S3PutProperties *putProperties, S3PutObjectHandler *handler, int seq, const char *upload_id, int partContentLength, - S3RequestContext *requestContext, void *callbackData) + S3RequestContext *requestContext, + int timeoutMs, + void *callbackData) { char queryParams[512]; snprintf(queryParams, 512, "partNumber=%d&uploadId=%s", seq, upload_id); @@ -222,7 +226,7 @@ void S3_upload_part(S3BucketContext *bucketContext, const char *key, 0, // fromS3Callback handler->responseHandler.completeCallback, // completeCallback callbackData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; request_perform(¶ms, requestContext); @@ -321,6 +325,7 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, S3MultipartCommitHandler *handler, const char *upload_id, int contentLength, S3RequestContext *requestContext, + int timeoutMs, void *callbackData) { char queryParams[512]; @@ -361,7 +366,7 @@ void S3_complete_multipart_upload(S3BucketContext *bucketContext, commitMultipartCallback, // fromS3Callback commitMultipartCompleteCallback, // completeCallback data, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; request_perform(¶ms, requestContext); @@ -839,6 +844,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, const char *uploadidmarker, const char *encodingtype, const char *delimiter, int maxuploads, S3RequestContext *requestContext, + int timeoutMs, const S3ListMultipartUploadsHandler *handler, void *callbackData) { @@ -955,7 +961,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, &listMultipartDataCallback, // fromS3Callback &listMultipartCompleteCallback, // completeCallback lmData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -967,6 +973,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, const char *partnumbermarker, const char *uploadid, const char *encodingtype, int maxparts, S3RequestContext *requestContext, + int timeoutMs, const S3ListPartsHandler *handler, void *callbackData) { // Compose the query params @@ -1078,7 +1085,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, &listPartsDataCallback, // fromS3Callback &listPartsCompleteCallback, // completeCallback lpData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/object.c b/src/object.c index cdfdbf6..a852519 100644 --- a/src/object.c +++ b/src/object.c @@ -175,6 +175,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, const char *key, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { /* Use the range copier with 0 length */ @@ -185,6 +186,7 @@ void S3_copy_object(const S3BucketContext *bucketContext, const char *key, putProperties, lastModifiedReturn, eTagReturnSize, eTagReturn, requestContext, + timeoutMs, handler, callbackData); } @@ -197,6 +199,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, const S3PutProperties *putProperties, int64_t *lastModifiedReturn, int eTagReturnSize, char *eTagReturn, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Create the callback data @@ -258,7 +261,7 @@ void S3_copy_object_range(const S3BucketContext *bucketContext, const char *key, ©ObjectDataCallback, // fromS3Callback ©ObjectCompleteCallback, // completeCallback data, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -356,6 +359,7 @@ void S3_head_object(const S3BucketContext *bucketContext, const char *key, void S3_delete_object(const S3BucketContext *bucketContext, const char *key, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { // Set up the RequestParams @@ -385,7 +389,7 @@ void S3_delete_object(const S3BucketContext *bucketContext, const char *key, 0, // fromS3Callback handler->completeCallback, // completeCallback callbackData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/request.c b/src/request.c index be431c0..a750925 100644 --- a/src/request.c +++ b/src/request.c @@ -1738,7 +1738,7 @@ S3Status S3_generate_authenticated_query_string RequestParams params = { http_request_method_to_type(httpMethod), *bucketContext, key, NULL, resource, - NULL, NULL, NULL, 0, 0, NULL, NULL, NULL, 0, NULL, NULL, NULL }; + NULL, NULL, NULL, 0, 0, NULL, NULL, NULL, 0, NULL, NULL, NULL, 0}; RequestComputedValues computed; S3Status status = setup_request(¶ms, &computed, 1); diff --git a/src/s3.c b/src/s3.c index 39d56f7..8754268 100644 --- a/src/s3.c +++ b/src/s3.c @@ -63,6 +63,7 @@ static int showResponsePropertiesG = 0; static S3Protocol protocolG = S3ProtocolHTTPS; static S3UriStyle uriStyleG = S3UriStylePath; static int retriesG = 5; +static int timeoutMsG = 0; static int verifyPeerG = 0; static const char *awsRegionG = NULL; @@ -203,6 +204,8 @@ static void usageExit(FILE *out) " -s/--show-properties : show response properties on stdout\n" " -r/--retries : retry retryable failures this number of times\n" " (default is 5)\n" +" -t/--timeout : request timeout, milliseconds. 0 if waiting forever\n" +" (default is 0)\n" " -v/--verify-peer : verify peer SSL certificate (default is no)\n" " -g/--region : use for request authorization\n" "\n" @@ -747,6 +750,7 @@ static struct option longOptionsG[] = { "unencrypted", no_argument, 0, 'u' }, { "show-properties", no_argument, 0, 's' }, { "retries", required_argument, 0, 'r' }, + { "timeout", required_argument, 0, 't' }, { "verify-peer", no_argument, 0, 'v' }, { "region", required_argument, 0, 'g' }, { 0, 0, 0, 0 } @@ -921,7 +925,7 @@ static void list_service(int allDetails) do { S3_list_service(protocolG, accessKeyIdG, secretAccessKeyG, 0, 0, - awsRegionG, 0, &listServiceHandler, &data); + awsRegionG, 0, timeoutMsG, &listServiceHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -965,7 +969,7 @@ static void test_bucket(int argc, char **argv, int optindex) do { S3_test_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, 0, 0, bucketName, awsRegionG, sizeof(locationConstraint), - locationConstraint, 0, &responseHandler, 0); + locationConstraint, 0, timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); const char *result; @@ -1103,7 +1107,7 @@ static void delete_bucket(int argc, char **argv, int optindex) do { S3_delete_bucket(protocolG, uriStyleG, accessKeyIdG, secretAccessKeyG, - 0, 0, bucketName, awsRegionG, 0, &responseHandler, 0); + 0, 0, bucketName, awsRegionG, 0, timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -1284,7 +1288,7 @@ static void list_bucket(const char *bucketName, const char *prefix, data.isTruncated = 0; do { S3_list_bucket(&bucketContext, prefix, data.nextMarker, - delimiter, maxkeys, 0, &listBucketHandler, &data); + delimiter, maxkeys, 0, timeoutMsG, &listBucketHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { break; @@ -1719,6 +1723,7 @@ static void list_multipart_uploads(int argc, char **argv, int optindex) data.nextKeyMarker, data.nextUploadIdMarker, encodingtype, delimiter, maxuploads, 0, + timeoutMsG, &listMultipartUploadsHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -1839,7 +1844,8 @@ static void list_parts(int argc, char **argv, int optindex) S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, uploadid, encodingtype, maxparts, - 0, &listPartsHandler, &data); + 0, timeoutMsG, + &listPartsHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { break; @@ -1937,7 +1943,7 @@ static void abort_multipart_upload(int argc, char **argv, int optindex) do { S3_abort_multipart_upload(&bucketContext, key, uploadid, - &abortMultipartUploadHandler); + timeoutMsG, &abortMultipartUploadHandler); } while (S3_status_is_retryable(statusG) && should_retry()); S3_deinitialize(); @@ -1984,7 +1990,7 @@ static void delete_object(int argc, char **argv, int optindex) }; do { - S3_delete_object(&bucketContext, key, 0, &responseHandler, 0); + S3_delete_object(&bucketContext, key, 0, timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if ((statusG != S3StatusOK) && @@ -2123,7 +2129,7 @@ static int try_get_parts_info(const char *bucketName, const char *key, data.isTruncated = 0; do { S3_list_parts(&bucketContext, key, data.nextPartNumberMarker, - manager->upload_id, 0, 0, 0, &listPartsHandler, + manager->upload_id, 0, 0, 0, timeoutMsG, &listPartsHandler, &data); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -2462,7 +2468,7 @@ static void put_object(int argc, char **argv, int optindex, } do { - S3_initiate_multipart(&bucketContext, key,0, &handler,0, &manager); + S3_initiate_multipart(&bucketContext, key,0, &handler,0, timeoutMsG, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); if (manager.upload_id == 0 || statusG != S3StatusOK) { @@ -2513,11 +2519,14 @@ static void put_object(int argc, char **argv, int optindex, &putProperties, &lastModified, 512 /*TBD - magic # */, manager.etags[seq-1], 0, + timeoutMsG, ©ResponseHandler, 0); } else { S3_upload_part(&bucketContext, key, &putProperties, &putObjectHandler, seq, manager.upload_id, - partContentLength,0, &partData); + partContentLength, + 0, timeoutMsG, + &partData); } } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -2546,7 +2555,7 @@ static void put_object(int argc, char **argv, int optindex, do { S3_complete_multipart_upload(&bucketContext, key, &commit_handler, manager.upload_id, manager.remaining, - 0, &manager); + 0, timeoutMsG, &manager); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { printError(); @@ -2643,7 +2652,8 @@ static void copy_object(int argc, char **argv, int optindex) // Find size of existing key to determine if MP required do { S3_list_bucket(&listBucketContext, sourceKey, NULL, - ".", 1, 0, &listBucketHandler, &sourceSize); + ".", 1, 0, + timeoutMsG, &listBucketHandler, &sourceSize); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { fprintf(stderr, "\nERROR: Unable to get source object size (%s)\n", @@ -2811,6 +2821,7 @@ static void copy_object(int argc, char **argv, int optindex) S3_copy_object(&bucketContext, sourceKey, destinationBucketName, destinationKey, anyPropertiesSet ? &putProperties : 0, &lastModified, sizeof(eTag), eTag, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -3240,7 +3251,8 @@ void get_acl(int argc, char **argv, int optindex) do { S3_get_acl(&bucketContext, key, ownerId, ownerDisplayName, - &aclGrantCount, aclGrants, 0, &responseHandler, 0); + &aclGrantCount, aclGrants, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG == S3StatusOK) { @@ -3405,7 +3417,8 @@ void set_acl(int argc, char **argv, int optindex) do { S3_set_acl(&bucketContext, key, ownerId, ownerDisplayName, - aclGrantCount, aclGrants, 0, &responseHandler, 0); + aclGrantCount, aclGrants, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -3500,6 +3513,7 @@ void get_logging(int argc, char **argv, int optindex) do { S3_get_server_access_logging(&bucketContext, targetBucket, targetPrefix, &aclGrantCount, aclGrants, 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); @@ -3668,7 +3682,8 @@ void set_logging(int argc, char **argv, int optindex) do { S3_set_server_access_logging(&bucketContext, targetBucket, targetPrefix, aclGrantCount, aclGrants, - 0, &responseHandler, 0); + 0, + timeoutMsG, &responseHandler, 0); } while (S3_status_is_retryable(statusG) && should_retry()); if (statusG != S3StatusOK) { @@ -3686,7 +3701,7 @@ int main(int argc, char **argv) // Parse args while (1) { int idx = 0; - int c = getopt_long(argc, argv, "vfhusr:g:", longOptionsG, &idx); + int c = getopt_long(argc, argv, "vfhusr:t:g:", longOptionsG, &idx); if (c == -1) { // End of options @@ -3716,6 +3731,16 @@ int main(int argc, char **argv) } } break; + case 't': { + const char *v = optarg; + timeoutMsG = 0; + while (*v) { + timeoutMsG *= 10; + timeoutMsG += *v - '0'; + v++; + } + } + break; case 'v': verifyPeerG = S3_INIT_VERIFY_PEER; break; diff --git a/src/service.c b/src/service.c index 2414513..7c3d785 100644 --- a/src/service.c +++ b/src/service.c @@ -134,6 +134,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *authRegion, S3RequestContext *requestContext, + int timeoutMs, const S3ListServiceHandler *handler, void *callbackData) { // Create and set up the callback data @@ -185,7 +186,7 @@ void S3_list_service(S3Protocol protocol, const char *accessKeyId, &dataCallback, // fromS3Callback &completeCallback, // completeCallback data, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request diff --git a/src/service_access_logging.c b/src/service_access_logging.c index 49919c3..141db91 100644 --- a/src/service_access_logging.c +++ b/src/service_access_logging.c @@ -298,6 +298,7 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, int *aclGrantCountReturn, S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { @@ -346,7 +347,7 @@ void S3_get_server_access_logging(const S3BucketContext *bucketContext, &getBlsDataCallback, // fromS3Callback &getBlsCompleteCallback, // completeCallback gsData, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request @@ -494,6 +495,7 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, const char *targetPrefix, int aclGrantCount, const S3AclGrant *aclGrants, S3RequestContext *requestContext, + int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { @@ -553,7 +555,7 @@ void S3_set_server_access_logging(const S3BucketContext *bucketContext, 0, // fromS3Callback &setSalCompleteCallback, // completeCallback data, // callbackData - 0 // timeoutMs + timeoutMs // timeoutMs }; // Perform the request From 09c78629076c5b309e726e65f3ed1a6a2c93f05e Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 3 Feb 2017 18:14:08 +0300 Subject: [PATCH 30/57] version: set version 4.0 because request timeout has been added --- GNUmakefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index ab64908..88e2447 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -1,9 +1,9 @@ # GNUmakefile -# +# # Copyright 2008 Bryan Ischo -# +# # This file is part of libs3. -# +# # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free # Software Foundation, version 3 of the License. @@ -38,7 +38,7 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 3 +LIBS3_VER_MAJOR ?= 4 LIBS3_VER_MINOR ?= 0 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) @@ -221,7 +221,7 @@ $(BUILD)/obj/%.do: src/%.c @ $(CC) $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< + $(VERBOSE_SHOW) $(CC) $(CFLAGS) -fpic -fPIC -o $@ -c $< # -------------------------------------------------------------------------- From c097c82486195352eff07c9ef8f38091768ba66a Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Mon, 6 Feb 2017 10:53:50 +0300 Subject: [PATCH 31/57] Fix comment for request timeout --- inc/request.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/inc/request.h b/inc/request.h index 041d548..0143af7 100644 --- a/inc/request.h +++ b/inc/request.h @@ -1,10 +1,10 @@ /** ************************************************************************** * request.h - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -103,7 +103,7 @@ typedef struct RequestParams // Data passed to the callbacks void *callbackData; - // Request timeout. 0 if not set + // Request timeout. If 0, no timeout will be enforced int timeoutMs; } RequestParams; From 24825a855c9c094bd75580456a5047d869f6061d Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Wed, 8 Feb 2017 16:05:33 +0300 Subject: [PATCH 32/57] lifecycle: implement lifecycle support --- inc/libs3.h | 49 ++++++++++ src/acl.c | 252 +++++++++++++++++++++++++++++++++++++++++++++++++- src/general.c | 29 +++--- src/request.c | 3 + src/s3.c | 191 +++++++++++++++++++++++++++++++++++++- 5 files changed, 504 insertions(+), 20 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index fcfe224..f48eaa8 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -298,6 +298,7 @@ typedef enum S3StatusServerFailedVerification , S3StatusConnectionFailed , S3StatusAbortedByCallback , + S3StatusNotSupported , /** * Errors from the S3 service @@ -2195,6 +2196,54 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, const S3ResponseHandler *handler, void *callbackData); +/** ************************************************************************** + * Lifecycle Control Functions + ************************************************************************** **/ + +/** + * Gets the lifecycle for the given bucket + * + * @param bucketContext gives the bucket and associated parameters for this + * request + * @param lifecycleXmlDocumentReturn buffer for lifecycle XML document + * @param lifecycleXmlDocumentBufferSize size of the buffer + * @param requestContext if non-NULL, gives the S3RequestContext to add this + * request to, and does not perform the request immediately. If NULL, + * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds + * @param handler gives the callbacks to call as the request is processed and + * completed + * @param callbackData will be passed in as the callbackData parameter to + * all callbacks for this request + **/ +void S3_get_lifecycle(const S3BucketContext *bucketContext, + char *lifecycleXmlDocumentReturn, int lifecycleXmlDocumentBufferSize, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData); + + +/** + * Sets the lifecycle for the given bucket + * + * @param bucketContext gives the bucket and associated parameters for this + * request + * @param lifecycleXmlDocument Lifecycle configuration as an XML document + * @param requestContext if non-NULL, gives the S3RequestContext to add this + * request to, and does not perform the request immediately. If NULL, + * performs the request immediately and synchronously. + * @param timeoutMs if not 0 contains total request timeout in milliseconds + * @param handler gives the callbacks to call as the request is processed and + * completed + * @param callbackData will be passed in as the callbackData parameter to + * all callbacks for this request + **/ +void S3_set_lifecycle(const S3BucketContext *bucketContext, + const char *lifecycleXmlDocument, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData); + /** ************************************************************************** * Server Access Log Functions ************************************************************************** **/ diff --git a/src/acl.c b/src/acl.c index 04d977c..11af49e 100644 --- a/src/acl.c +++ b/src/acl.c @@ -26,6 +26,14 @@ #include #include + +#ifndef __APPLE__ + #include + #include + #include + #include +#endif + #include "libs3.h" #include "request.h" @@ -234,7 +242,7 @@ typedef struct SetAclData void *callbackData; int aclXmlDocumentLen; - char aclXmlDocument[ACL_XML_DOC_MAXSIZE]; + const char *aclXmlDocument; int aclXmlDocumentBytesWritten; } SetAclData; @@ -292,6 +300,8 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, int timeoutMs, const S3ResponseHandler *handler, void *callbackData) { + char aclBuffer[ACL_XML_DOC_MAXSIZE]; + if (aclGrantCount > S3_MAX_ACL_GRANT_COUNT) { (*(handler->completeCallback)) (S3StatusTooManyGrants, 0, callbackData); @@ -304,11 +314,13 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, return; } + data->aclXmlDocument = aclBuffer; + // Convert aclGrants to XML document S3Status status = generateAclXmlDocument (ownerId, ownerDisplayName, aclGrantCount, aclGrants, - &(data->aclXmlDocumentLen), data->aclXmlDocument, - sizeof(data->aclXmlDocument)); + &(data->aclXmlDocumentLen), aclBuffer, + sizeof(aclBuffer)); if (status != S3StatusOK) { free(data); (*(handler->completeCallback))(status, 0, callbackData); @@ -354,3 +366,237 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, // Perform the request request_perform(¶ms, requestContext); } + + +// get lifecycle ------------------------------------------------------------------- + +typedef struct GetLifecycleData +{ + S3ResponsePropertiesCallback *responsePropertiesCallback; + S3ResponseCompleteCallback *responseCompleteCallback; + void *callbackData; + + char *lifecycleXmlDocumentReturn; + int lifecycleXmlDocumentBufferSize; + int lifecycleXmlDocumentWritten; +} GetLifecycleData; + + +static S3Status getLifecyclePropertiesCallback + (const S3ResponseProperties *responseProperties, void *callbackData) +{ + GetLifecycleData *gaData = (GetLifecycleData *) callbackData; + + return (*(gaData->responsePropertiesCallback)) + (responseProperties, gaData->callbackData); +} + + +static S3Status getLifecycleDataCallback(int bufferSize, const char *buffer, + void *callbackData) +{ + GetLifecycleData *gaData = (GetLifecycleData *) callbackData; + + if ((gaData->lifecycleXmlDocumentWritten + bufferSize) >= gaData->lifecycleXmlDocumentBufferSize) + return S3StatusXmlDocumentTooLarge; + + snprintf(gaData->lifecycleXmlDocumentReturn + gaData->lifecycleXmlDocumentWritten, bufferSize + 1, "%s", buffer); + gaData->lifecycleXmlDocumentWritten += bufferSize; + + return S3StatusOK; +} + + +static void getLifecycleCompleteCallback(S3Status requestStatus, + const S3ErrorDetails *s3ErrorDetails, + void *callbackData) +{ + GetLifecycleData *gaData = (GetLifecycleData *) callbackData; + + (*(gaData->responseCompleteCallback)) + (requestStatus, s3ErrorDetails, gaData->callbackData); + + free(gaData); +} + + +void S3_get_lifecycle(const S3BucketContext *bucketContext, + char *lifecycleXmlDocumentReturn, int lifecycleXmlDocumentBufferSize, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData) +{ + // Create the callback data + GetLifecycleData *gaData = (GetLifecycleData *) malloc(sizeof(GetLifecycleData)); + if (!gaData) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + gaData->responsePropertiesCallback = handler->propertiesCallback; + gaData->responseCompleteCallback = handler->completeCallback; + gaData->callbackData = callbackData; + + gaData->lifecycleXmlDocumentReturn = lifecycleXmlDocumentReturn; + gaData->lifecycleXmlDocumentBufferSize = lifecycleXmlDocumentBufferSize; + gaData->lifecycleXmlDocumentWritten = 0; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypeGET, // httpRequestType + { bucketContext->hostName, // hostName + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion + 0, // key + 0, // queryParams + "lifecycle", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + 0, // putProperties + &getLifecyclePropertiesCallback, // propertiesCallback + 0, // toS3Callback + 0, // toS3CallbackTotalSize + &getLifecycleDataCallback, // fromS3Callback + &getLifecycleCompleteCallback, // completeCallback + gaData, // callbackData + timeoutMs // timeoutMs + }; + + // Perform the request + request_perform(¶ms, requestContext); +} + + +#ifndef __APPLE__ +// Calculate MD5 and encode it as base64 +void generate_content_md5(const char* data, int size, + char* retBuffer, int retBufferSize) { + MD5_CTX mdContext; + BIO *bio, *b64; + BUF_MEM *bufferPtr; + + char md5Buffer[MD5_DIGEST_LENGTH]; + + MD5_Init(&mdContext); + MD5_Update(&mdContext, data, size); + MD5_Final((unsigned char*)md5Buffer, &mdContext); + + + b64 = BIO_new(BIO_f_base64()); + bio = BIO_new(BIO_s_mem()); + bio = BIO_push(b64, bio); + + BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); //Ignore newlines - write everything in one line + BIO_write(bio, md5Buffer, sizeof(md5Buffer)); + BIO_flush(bio); + BIO_get_mem_ptr(bio, &bufferPtr); + BIO_set_close(bio, BIO_NOCLOSE); + + if ((unsigned int)retBufferSize + 1 < bufferPtr->length) { + retBuffer[0] = '\0'; + BIO_free_all(bio); + return; + } + + memcpy(retBuffer, bufferPtr->data, bufferPtr->length); + retBuffer[bufferPtr->length] = '\0'; + + BIO_free_all(bio); +} +#endif + + +void S3_set_lifecycle(const S3BucketContext *bucketContext, + const char *lifecycleXmlDocument, + S3RequestContext *requestContext, + int timeoutMs, + const S3ResponseHandler *handler, void *callbackData) +{ +#ifdef __APPLE__ + /* This request requires calculating MD5 sum. + * MD5 sum requires OpenSSL library, which is not used on Apple. + * TODO Implement some MD5+Base64 caculation on Apple + */ + (*(handler->completeCallback))(S3StatusNotSupported, 0, callbackData); + return; +#else + char md5Base64[MD5_DIGEST_LENGTH * 2]; + + SetAclData *data = (SetAclData *) malloc(sizeof(SetAclData)); + if (!data) { + (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); + return; + } + + + data->aclXmlDocument = lifecycleXmlDocument; + data->aclXmlDocumentLen = strlen(lifecycleXmlDocument); + + data->responsePropertiesCallback = handler->propertiesCallback; + data->responseCompleteCallback = handler->completeCallback; + data->callbackData = callbackData; + + data->aclXmlDocumentBytesWritten = 0; + + generate_content_md5(data->aclXmlDocument, data->aclXmlDocumentLen, + md5Base64, sizeof (md5Base64)); + + // Set up S3PutProperties + S3PutProperties properties = + { + 0, // contentType + md5Base64, // md5 + 0, // cacheControl + 0, // contentDispositionFilename + 0, // contentEncoding + -1, // expires + 0, // cannedAcl + 0, // metaDataCount + 0, // metaData + 0 // useServerSideEncryption + }; + + // Set up the RequestParams + RequestParams params = + { + HttpRequestTypePUT, // httpRequestType + { bucketContext->hostName, // hostName + bucketContext->bucketName, // bucketName + bucketContext->protocol, // protocol + bucketContext->uriStyle, // uriStyle + bucketContext->accessKeyId, // accessKeyId + bucketContext->secretAccessKey, // secretAccessKey + bucketContext->securityToken, // securityToken + bucketContext->authRegion }, // authRegion + 0, // key + 0, // queryParams + "lifecycle", // subResource + 0, // copySourceBucketName + 0, // copySourceKey + 0, // getConditions + 0, // startByte + 0, // byteCount + &properties, // putProperties + &setAclPropertiesCallback, // propertiesCallback + &setAclDataCallback, // toS3Callback + data->aclXmlDocumentLen, // toS3CallbackTotalSize + 0, // fromS3Callback + &setAclCompleteCallback, // completeCallback + data, // callbackData + timeoutMs // timeoutMs + }; + + // Perform the request + request_perform(¶ms, requestContext); +#endif +} + diff --git a/src/general.c b/src/general.c index fa55939..df13225 100644 --- a/src/general.c +++ b/src/general.c @@ -1,10 +1,10 @@ /** ************************************************************************** * general.c - * + * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free * Software Foundation, version 3 of the License. @@ -107,6 +107,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ServerFailedVerification); handlecase(ConnectionFailed); handlecase(AbortedByCallback); + handlecase(NotSupported); handlecase(ErrorAccessDenied); handlecase(ErrorAccountProblem); handlecase(ErrorAmbiguousGrantByEmailAddress); @@ -119,7 +120,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorEntityTooSmall); handlecase(ErrorEntityTooLarge); handlecase(ErrorExpiredToken); - handlecase(ErrorIllegalVersioningConfigurationException); + handlecase(ErrorIllegalVersioningConfigurationException); handlecase(ErrorIncompleteBody); handlecase(ErrorIncorrectNumberOfFilesInPostRequest); handlecase(ErrorInlineDataTooLarge); @@ -131,7 +132,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorInvalidBucketState); handlecase(ErrorInvalidDigest); handlecase(ErrorInvalidLocationConstraint); - handlecase(ErrorInvalidObjectState); + handlecase(ErrorInvalidObjectState); handlecase(ErrorInvalidPart); handlecase(ErrorInvalidPartOrder); handlecase(ErrorInvalidPayer); @@ -161,8 +162,8 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorNoSuchBucket); handlecase(ErrorNoSuchKey); handlecase(ErrorNoSuchLifecycleConfiguration); - handlecase(ErrorNoSuchUpload); - handlecase(ErrorNoSuchVersion); + handlecase(ErrorNoSuchUpload); + handlecase(ErrorNoSuchVersion); handlecase(ErrorNotImplemented); handlecase(ErrorNotSignedUp); handlecase(ErrorNotSuchBucketPolicy); @@ -295,7 +296,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, if (data) { if (!strcmp(elementPath, "AccessControlPolicy/Owner/ID")) { - caData->ownerIdLen += + caData->ownerIdLen += snprintf(&(caData->ownerId[caData->ownerIdLen]), S3_MAX_GRANTEE_USER_ID_SIZE - caData->ownerIdLen - 1, "%.*s", dataLen, data); @@ -305,18 +306,18 @@ static S3Status convertAclXmlCallback(const char *elementPath, } else if (!strcmp(elementPath, "AccessControlPolicy/Owner/" "DisplayName")) { - caData->ownerDisplayNameLen += + caData->ownerDisplayNameLen += snprintf(&(caData->ownerDisplayName [caData->ownerDisplayNameLen]), S3_MAX_GRANTEE_DISPLAY_NAME_SIZE - - caData->ownerDisplayNameLen - 1, + caData->ownerDisplayNameLen - 1, "%.*s", dataLen, data); - if (caData->ownerDisplayNameLen >= + if (caData->ownerDisplayNameLen >= S3_MAX_GRANTEE_DISPLAY_NAME_SIZE) { return S3StatusUserDisplayNameTooLong; } } - else if (!strcmp(elementPath, + else if (!strcmp(elementPath, "AccessControlPolicy/AccessControlList/Grant/" "Grantee/EmailAddress")) { // AmazonCustomerByEmail @@ -382,7 +383,7 @@ static S3Status convertAclXmlCallback(const char *elementPath, else if (caData->userId[0] && caData->userDisplayName[0]) { grant->granteeType = S3GranteeTypeCanonicalUser; strcpy(grant->grantee.canonicalUser.id, caData->userId); - strcpy(grant->grantee.canonicalUser.displayName, + strcpy(grant->grantee.canonicalUser.displayName, caData->userDisplayName); } else if (caData->groupUri[0]) { @@ -466,7 +467,7 @@ S3Status S3_convert_acl(char *aclXml, char *ownerId, char *ownerDisplayName, S3Status status = simplexml_add(&simpleXml, aclXml, strlen(aclXml)); simplexml_deinitialize(&simpleXml); - + return status; } diff --git a/src/request.c b/src/request.c index a750925..e1aad97 100644 --- a/src/request.c +++ b/src/request.c @@ -703,6 +703,9 @@ static void canonicalize_signature_headers(RequestComputedValues *values) if (values->rangeHeader[0]) { sortedHeaders[headerCount++] = values->rangeHeader; } + if (values->md5Header[0]) { + sortedHeaders[headerCount++] = values->md5Header; + } // Now sort these kv_gnome_sort(sortedHeaders, headerCount, ':'); diff --git a/src/s3.c b/src/s3.c index 8754268..b8d4405 100644 --- a/src/s3.c +++ b/src/s3.c @@ -251,17 +251,24 @@ static void usageExit(FILE *out) " setacl : Set the ACL of a bucket or key\n" " [/] : Bucket or bucket/key to set the ACL of\n" " [filename] : Input filename for ACL (default is stdin)\n" +" getlifecycle : Get the lifecycle of a bucket\n" +" : Bucket or bucket to get the lifecycle of\n" +" [filename] : Output filename for lifecycle (default is stdout)\n" +"\n" +" setlifecycle : Set the lifecycle of a bucket or key\n" +" : Bucket or bucket to set the lifecycle of\n" +" [filename] : Input filename for lifecycle (default is stdin)\n" "\n" " getlogging : Get the logging status of a bucket\n" " : Bucket to get the logging status of\n" -" [filename] : Output filename for ACL (default is stdout)\n" +" [filename] : Output filename for logging (default is stdout)\n" "\n" " setlogging : Set the logging status of a bucket\n" " : Bucket to set the logging status of\n" " [targetBucket] : Target bucket to log to; if not present, disables\n" " logging\n" " [targetPrefix] : Key prefix to use for logs\n" -" [filename] : Input filename for ACL (default is stdin)\n" +" [filename] : Input filename for logging (default is stdin)\n" "\n" " put : Puts an object\n" " / : Bucket/key to put object to\n" @@ -3381,7 +3388,7 @@ void set_acl(int argc, char **argv, int optindex) // Read in the complete ACL char aclBuf[65536]; - aclBuf[fread(aclBuf, 1, sizeof(aclBuf), infile)] = 0; + aclBuf[fread(aclBuf, 1, sizeof(aclBuf) - 1, infile)] = 0; char ownerId[S3_MAX_GRANTEE_USER_ID_SIZE]; char ownerDisplayName[S3_MAX_GRANTEE_DISPLAY_NAME_SIZE]; @@ -3430,6 +3437,178 @@ void set_acl(int argc, char **argv, int optindex) S3_deinitialize(); } +// get lifecycle ------------------------------------------------------------------- + +void get_lifecycle(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex++]; + + const char *filename = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { + filename = &(param[FILENAME_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + FILE *outfile = 0; + + if (filename) { + // Stat the file, and if it doesn't exist, open it in w mode + struct stat buf; + if (stat(filename, &buf) == -1) { + outfile = fopen(filename, "w" FOPEN_EXTRA_FLAGS); + } + else { + // Open in r+ so that we don't truncate the file, just in case + // there is an error and we write no bytes, we leave the file + // unmodified + outfile = fopen(filename, "r+" FOPEN_EXTRA_FLAGS); + } + + if (!outfile) { + fprintf(stderr, "\nERROR: Failed to open output file %s: ", + filename); + perror(0); + exit(-1); + } + } + else if (showResponsePropertiesG) { + fprintf(stderr, "\nERROR: getlifecycle -s requires a filename parameter\n"); + usageExit(stderr); + } + else { + outfile = stdout; + } + + char lifecycleBuffer[64 * 1024]; + + S3_init(); + + S3BucketContext bucketContext = + { + 0, + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG, + 0, + awsRegionG + }; + + S3ResponseHandler responseHandler = + { + &responsePropertiesCallback, + &responseCompleteCallback + }; + + do { + S3_get_lifecycle(&bucketContext, + lifecycleBuffer, sizeof(lifecycleBuffer), + 0, timeoutMsG, &responseHandler, 0); + } while (S3_status_is_retryable(statusG) && should_retry()); + + if (statusG == S3StatusOK) { + fprintf(outfile, "%s", lifecycleBuffer); + } + else { + printError(); + } + + fclose(outfile); + + S3_deinitialize(); +} + + +// set lifecycle ------------------------------------------------------------------- + +void set_lifecycle(int argc, char **argv, int optindex) +{ + if (optindex == argc) { + fprintf(stderr, "\nERROR: Missing parameter: bucket\n"); + usageExit(stderr); + } + + const char *bucketName = argv[optindex++]; + + const char *filename = 0; + + while (optindex < argc) { + char *param = argv[optindex++]; + if (!strncmp(param, FILENAME_PREFIX, FILENAME_PREFIX_LEN)) { + filename = &(param[FILENAME_PREFIX_LEN]); + } + else { + fprintf(stderr, "\nERROR: Unknown param: %s\n", param); + usageExit(stderr); + } + } + + FILE *infile; + + if (filename) { + if (!(infile = fopen(filename, "r" FOPEN_EXTRA_FLAGS))) { + fprintf(stderr, "\nERROR: Failed to open input file %s: ", + filename); + perror(0); + exit(-1); + } + } + else { + infile = stdin; + } + + // Read in the complete ACL + char lifecycleBuf[65536]; + lifecycleBuf[fread(lifecycleBuf, 1, sizeof(lifecycleBuf) - 1, infile)] = 0; + + S3_init(); + + S3BucketContext bucketContext = + { + 0, + bucketName, + protocolG, + uriStyleG, + accessKeyIdG, + secretAccessKeyG, + 0, + awsRegionG + }; + + S3ResponseHandler responseHandler = + { + &responsePropertiesCallback, + &responseCompleteCallback + }; + + do { + S3_set_lifecycle(&bucketContext, + lifecycleBuf, + 0, timeoutMsG, &responseHandler, 0); + } while (S3_status_is_retryable(statusG) && should_retry()); + + if (statusG != S3StatusOK) { + printError(); + } + + fclose(infile); + + S3_deinitialize(); +} + // get logging ---------------------------------------------------------------- @@ -3831,6 +4010,12 @@ int main(int argc, char **argv) else if (!strcmp(command, "setacl")) { set_acl(argc, argv, optind); } + else if (!strcmp(command, "getlifecycle")) { + get_lifecycle(argc, argv, optind); + } + else if (!strcmp(command, "setlifecycle")) { + set_lifecycle(argc, argv, optind); + } else if (!strcmp(command, "getlogging")) { get_logging(argc, argv, optind); } From aa5b5fdacaf9a773590be7c94fb83e619741680c Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Wed, 8 Feb 2017 17:41:53 +0300 Subject: [PATCH 33/57] lifecycle: fix naming --- GNUmakefile | 2 +- GNUmakefile.mingw | 8 ++-- GNUmakefile.osx | 10 ++-- src/{acl.c => bucket_metadata.c} | 80 ++++++++++++++++---------------- 4 files changed, 50 insertions(+), 50 deletions(-) rename src/{acl.c => bucket_metadata.c} (90%) diff --git a/GNUmakefile b/GNUmakefile index 88e2447..f00b804 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -233,7 +233,7 @@ LIBS3_STATIC = $(BUILD)/lib/libs3.a .PHONY: libs3 libs3: $(LIBS3_SHARED) $(LIBS3_STATIC) -LIBS3_SOURCES := acl.c bucket.c error_parser.c general.c \ +LIBS3_SOURCES := bucket.c bucket_metadata.c error_parser.c general.c \ object.c request.c request_context.c \ response_headers_handler.c service_access_logging.c \ service.c simplexml.c util.c multipart.c diff --git a/GNUmakefile.mingw b/GNUmakefile.mingw index b2712f4..99224ab 100644 --- a/GNUmakefile.mingw +++ b/GNUmakefile.mingw @@ -1,9 +1,9 @@ # GNUmakefile.mingw -# +# # Copyright 2008 Bryan Ischo -# +# # This file is part of libs3. -# +# # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free # Software Foundation, version 3 of the License. @@ -212,7 +212,7 @@ LIBS3_STATIC = $(BUILD)/lib/libs3.a .PHONY: libs3 libs3: $(LIBS3_SHARED) $(BUILD)/lib/libs3.a -LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \ +LIBS3_SOURCES := src/bucket.c src/bucket_metadata.c src/error_parser.c src/general.c \ src/object.c src/request.c src/request_context.c \ src/response_headers_handler.c src/service_access_logging.c \ src/service.c src/simplexml.c src/util.c src/multipart.c \ diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 25def23..7dc48b2 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -1,9 +1,9 @@ # GNUmakefile.osx -# +# # Copyright 2008 Bryan Ischo -# +# # This file is part of libs3. -# +# # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free # Software Foundation, version 3 of the License. @@ -211,7 +211,7 @@ $(BUILD)/obj/%.do: src/%.c @ gcc $(CFLAGS) -M -MG -MQ $@ -DCOMPILINGDEPENDENCIES \ -o $(BUILD)/dep/$(<:%.c=%.dd) -c $< @ mkdir -p $(dir $@) - $(VERBOSE_SHOW) gcc $(CFLAGS) -fpic -fPIC -o $@ -c $< + $(VERBOSE_SHOW) gcc $(CFLAGS) -fpic -fPIC -o $@ -c $< # -------------------------------------------------------------------------- @@ -223,7 +223,7 @@ LIBS3_STATIC = $(BUILD)/lib/libs3.a .PHONY: libs3 libs3: $(LIBS3_SHARED) $(LIBS3_SHARED_MAJOR) $(BUILD)/lib/libs3.a -LIBS3_SOURCES := src/acl.c src/bucket.c src/error_parser.c src/general.c \ +LIBS3_SOURCES := src/bucket.c src/bucket_metadata.c src/error_parser.c src/general.c \ src/object.c src/request.c src/request_context.c \ src/response_headers_handler.c src/service_access_logging.c \ src/service.c src/simplexml.c src/util.c src/multipart.c diff --git a/src/acl.c b/src/bucket_metadata.c similarity index 90% rename from src/acl.c rename to src/bucket_metadata.c index 11af49e..64c2659 100644 --- a/src/acl.c +++ b/src/bucket_metadata.c @@ -1,5 +1,5 @@ /** ************************************************************************** - * acl.c + * bucket_metadata.c * * Copyright 2008 Bryan Ischo * @@ -55,7 +55,7 @@ typedef struct GetAclData S3AclGrant *aclGrants; char *ownerId; char *ownerDisplayName; - string_buffer(aclXmlDocument, ACL_XML_DOC_MAXSIZE); + string_buffer(xmlDocument, ACL_XML_DOC_MAXSIZE); } GetAclData; @@ -76,7 +76,7 @@ static S3Status getAclDataCallback(int bufferSize, const char *buffer, int fit; - string_buffer_append(gaData->aclXmlDocument, buffer, bufferSize, fit); + string_buffer_append(gaData->xmlDocument, buffer, bufferSize, fit); return fit ? S3StatusOK : S3StatusXmlDocumentTooLarge; } @@ -91,7 +91,7 @@ static void getAclCompleteCallback(S3Status requestStatus, if (requestStatus == S3StatusOK) { // Parse the document requestStatus = S3_convert_acl - (gaData->aclXmlDocument, gaData->ownerId, gaData->ownerDisplayName, + (gaData->xmlDocument, gaData->ownerId, gaData->ownerDisplayName, gaData->aclGrantCountReturn, gaData->aclGrants); } @@ -124,7 +124,7 @@ void S3_get_acl(const S3BucketContext *bucketContext, const char *key, gaData->aclGrants = aclGrants; gaData->ownerId = ownerId; gaData->ownerDisplayName = ownerDisplayName; - string_buffer_initialize(gaData->aclXmlDocument); + string_buffer_initialize(gaData->xmlDocument); *aclGrantCountReturn = 0; // Set up the RequestParams @@ -235,35 +235,35 @@ static S3Status generateAclXmlDocument(const char *ownerId, } -typedef struct SetAclData +typedef struct SetXmlData { S3ResponsePropertiesCallback *responsePropertiesCallback; S3ResponseCompleteCallback *responseCompleteCallback; void *callbackData; - int aclXmlDocumentLen; - const char *aclXmlDocument; - int aclXmlDocumentBytesWritten; + int xmlDocumentLen; + const char *xmlDocument; + int xmlDocumentBytesWritten; -} SetAclData; +} SetXmlData; -static S3Status setAclPropertiesCallback +static S3Status setXmlPropertiesCallback (const S3ResponseProperties *responseProperties, void *callbackData) { - SetAclData *paData = (SetAclData *) callbackData; + SetXmlData *paData = (SetXmlData *) callbackData; return (*(paData->responsePropertiesCallback)) (responseProperties, paData->callbackData); } -static int setAclDataCallback(int bufferSize, char *buffer, void *callbackData) +static int setXmlDataCallback(int bufferSize, char *buffer, void *callbackData) { - SetAclData *paData = (SetAclData *) callbackData; + SetXmlData *paData = (SetXmlData *) callbackData; - int remaining = (paData->aclXmlDocumentLen - - paData->aclXmlDocumentBytesWritten); + int remaining = (paData->xmlDocumentLen - + paData->xmlDocumentBytesWritten); int toCopy = bufferSize > remaining ? remaining : bufferSize; @@ -271,20 +271,20 @@ static int setAclDataCallback(int bufferSize, char *buffer, void *callbackData) return 0; } - memcpy(buffer, &(paData->aclXmlDocument - [paData->aclXmlDocumentBytesWritten]), toCopy); + memcpy(buffer, &(paData->xmlDocument + [paData->xmlDocumentBytesWritten]), toCopy); - paData->aclXmlDocumentBytesWritten += toCopy; + paData->xmlDocumentBytesWritten += toCopy; return toCopy; } -static void setAclCompleteCallback(S3Status requestStatus, +static void setXmlCompleteCallback(S3Status requestStatus, const S3ErrorDetails *s3ErrorDetails, void *callbackData) { - SetAclData *paData = (SetAclData *) callbackData; + SetXmlData *paData = (SetXmlData *) callbackData; (*(paData->responseCompleteCallback)) (requestStatus, s3ErrorDetails, paData->callbackData); @@ -308,18 +308,18 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, return; } - SetAclData *data = (SetAclData *) malloc(sizeof(SetAclData)); + SetXmlData *data = (SetXmlData *) malloc(sizeof(SetXmlData)); if (!data) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); return; } - data->aclXmlDocument = aclBuffer; + data->xmlDocument = aclBuffer; // Convert aclGrants to XML document S3Status status = generateAclXmlDocument (ownerId, ownerDisplayName, aclGrantCount, aclGrants, - &(data->aclXmlDocumentLen), aclBuffer, + &(data->xmlDocumentLen), aclBuffer, sizeof(aclBuffer)); if (status != S3StatusOK) { free(data); @@ -331,7 +331,7 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, data->responseCompleteCallback = handler->completeCallback; data->callbackData = callbackData; - data->aclXmlDocumentBytesWritten = 0; + data->xmlDocumentBytesWritten = 0; // Set up the RequestParams RequestParams params = @@ -354,11 +354,11 @@ void S3_set_acl(const S3BucketContext *bucketContext, const char *key, 0, // startByte 0, // byteCount 0, // putProperties - &setAclPropertiesCallback, // propertiesCallback - &setAclDataCallback, // toS3Callback - data->aclXmlDocumentLen, // toS3CallbackTotalSize + &setXmlPropertiesCallback, // propertiesCallback + &setXmlDataCallback, // toS3Callback + data->xmlDocumentLen, // toS3CallbackTotalSize 0, // fromS3Callback - &setAclCompleteCallback, // completeCallback + &setXmlCompleteCallback, // completeCallback data, // callbackData timeoutMs // timeoutMs }; @@ -465,8 +465,8 @@ void S3_get_lifecycle(const S3BucketContext *bucketContext, &getLifecyclePropertiesCallback, // propertiesCallback 0, // toS3Callback 0, // toS3CallbackTotalSize - &getLifecycleDataCallback, // fromS3Callback - &getLifecycleCompleteCallback, // completeCallback + &getLifecycleDataCallback, // fromS3Callback + &getLifecycleCompleteCallback, // completeCallback gaData, // callbackData timeoutMs // timeoutMs }; @@ -531,23 +531,23 @@ void S3_set_lifecycle(const S3BucketContext *bucketContext, #else char md5Base64[MD5_DIGEST_LENGTH * 2]; - SetAclData *data = (SetAclData *) malloc(sizeof(SetAclData)); + SetXmlData *data = (SetXmlData *) malloc(sizeof(SetXmlData)); if (!data) { (*(handler->completeCallback))(S3StatusOutOfMemory, 0, callbackData); return; } - data->aclXmlDocument = lifecycleXmlDocument; - data->aclXmlDocumentLen = strlen(lifecycleXmlDocument); + data->xmlDocument = lifecycleXmlDocument; + data->xmlDocumentLen = strlen(lifecycleXmlDocument); data->responsePropertiesCallback = handler->propertiesCallback; data->responseCompleteCallback = handler->completeCallback; data->callbackData = callbackData; - data->aclXmlDocumentBytesWritten = 0; + data->xmlDocumentBytesWritten = 0; - generate_content_md5(data->aclXmlDocument, data->aclXmlDocumentLen, + generate_content_md5(data->xmlDocument, data->xmlDocumentLen, md5Base64, sizeof (md5Base64)); // Set up S3PutProperties @@ -586,11 +586,11 @@ void S3_set_lifecycle(const S3BucketContext *bucketContext, 0, // startByte 0, // byteCount &properties, // putProperties - &setAclPropertiesCallback, // propertiesCallback - &setAclDataCallback, // toS3Callback - data->aclXmlDocumentLen, // toS3CallbackTotalSize + &setXmlPropertiesCallback, // propertiesCallback + &setXmlDataCallback, // toS3Callback + data->xmlDocumentLen, // toS3CallbackTotalSize 0, // fromS3Callback - &setAclCompleteCallback, // completeCallback + &setXmlCompleteCallback, // completeCallback data, // callbackData timeoutMs // timeoutMs }; From 2abe6699e803b2cb7f079b7a18e9d13af7b3825f Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Wed, 8 Feb 2017 17:44:00 +0300 Subject: [PATCH 34/57] lifecycle: update version and TODO --- GNUmakefile | 2 +- GNUmakefile.mingw | 4 ++-- GNUmakefile.osx | 4 ++-- TODO | 5 ----- 4 files changed, 5 insertions(+), 10 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index f00b804..caab77e 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -39,7 +39,7 @@ # Set libs3 version number, unless it is already set. LIBS3_VER_MAJOR ?= 4 -LIBS3_VER_MINOR ?= 0 +LIBS3_VER_MINOR ?= 1 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) diff --git a/GNUmakefile.mingw b/GNUmakefile.mingw index 99224ab..ba1ad57 100644 --- a/GNUmakefile.mingw +++ b/GNUmakefile.mingw @@ -38,8 +38,8 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 3 -LIBS3_VER_MINOR ?= 0 +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) diff --git a/GNUmakefile.osx b/GNUmakefile.osx index 7dc48b2..a7de08f 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -38,8 +38,8 @@ # -------------------------------------------------------------------------- # Set libs3 version number, unless it is already set. -LIBS3_VER_MAJOR ?= 3 -LIBS3_VER_MINOR ?= 0 +LIBS3_VER_MAJOR ?= 4 +LIBS3_VER_MINOR ?= 1 LIBS3_VER := $(LIBS3_VER_MAJOR).$(LIBS3_VER_MINOR) diff --git a/TODO b/TODO index 5c05749..8508f2b 100644 --- a/TODO +++ b/TODO @@ -81,11 +81,6 @@ This is Amazon's "user" management API; not part of S3 although IAM users can be - POST /?delete -- 4 hours -=== Object Expiration Support === - -- Handle the /?lifecycle property: 8 hours - - === MFA Authentication === (part of Bucket Policy) From a99865aa69635eb8756e4f4795da928ef60e0107 Mon Sep 17 00:00:00 2001 From: Like Ma Date: Sat, 11 Mar 2017 05:34:28 +0800 Subject: [PATCH 35/57] Match http headers by case insensitive. --- src/response_headers_handler.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/response_headers_handler.c b/src/response_headers_handler.c index c5b5084..70046bc 100644 --- a/src/response_headers_handler.c +++ b/src/response_headers_handler.c @@ -26,6 +26,7 @@ #include #include +#include #include "response_headers_handler.h" @@ -113,44 +114,44 @@ void response_headers_handler_add(ResponseHeadersHandler *handler, int valuelen = (end - c) + 1, fit; - if (!strncmp(header, "x-amz-request-id", namelen)) { + if (!strncasecmp(header, "x-amz-request-id", namelen)) { responseProperties->requestId = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "x-amz-id-2", namelen)) { + else if (!strncasecmp(header, "x-amz-id-2", namelen)) { responseProperties->requestId2 = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "Content-Type", namelen)) { + else if (!strncasecmp(header, "Content-Type", namelen)) { responseProperties->contentType = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "Content-Length", namelen)) { + else if (!strncasecmp(header, "Content-Length", namelen)) { handler->responseProperties.contentLength = 0; while (*c) { handler->responseProperties.contentLength *= 10; handler->responseProperties.contentLength += (*c++ - '0'); } } - else if (!strncmp(header, "Server", namelen)) { + else if (!strncasecmp(header, "Server", namelen)) { responseProperties->server = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, "ETag", namelen)) { + else if (!strncasecmp(header, "ETag", namelen)) { responseProperties->eTag = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncmp(header, S3_METADATA_HEADER_NAME_PREFIX, + else if (!strncasecmp(header, S3_METADATA_HEADER_NAME_PREFIX, sizeof(S3_METADATA_HEADER_NAME_PREFIX) - 1)) { // Make sure there is room for another x-amz-meta header if (handler->responseProperties.metaDataCount == @@ -189,7 +190,7 @@ void response_headers_handler_add(ResponseHeadersHandler *handler, metaHeader->name = copiedName; metaHeader->value = copiedValue; } - else if (!strncmp(header, "x-amz-server-side-encryption", namelen)) { + else if (!strncasecmp(header, "x-amz-server-side-encryption", namelen)) { if (!strncmp(c, "AES256", sizeof("AES256") - 1)) { responseProperties->usesServerSideEncryption = 1; } From 89de6d6758380d5c5ac60fbf698a0a65ff33bea0 Mon Sep 17 00:00:00 2001 From: Like Ma Date: Fri, 10 Mar 2017 01:10:41 +0800 Subject: [PATCH 36/57] Fix building error in Ubuntu 12.04 Fix compilation error: value computed is not used [-Werror=unused-value] --- src/bucket_metadata.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bucket_metadata.c b/src/bucket_metadata.c index 64c2659..c2f1d0d 100644 --- a/src/bucket_metadata.c +++ b/src/bucket_metadata.c @@ -497,9 +497,9 @@ void generate_content_md5(const char* data, int size, BIO_set_flags(bio, BIO_FLAGS_BASE64_NO_NL); //Ignore newlines - write everything in one line BIO_write(bio, md5Buffer, sizeof(md5Buffer)); - BIO_flush(bio); + (void) BIO_flush(bio); BIO_get_mem_ptr(bio, &bufferPtr); - BIO_set_close(bio, BIO_NOCLOSE); + (void) BIO_set_close(bio, BIO_NOCLOSE); if ((unsigned int)retBufferSize + 1 < bufferPtr->length) { retBuffer[0] = '\0'; From 12ad1b397db2fb145211ca0766460a41e15c3322 Mon Sep 17 00:00:00 2001 From: Like Ma Date: Fri, 10 Mar 2017 13:45:43 +0800 Subject: [PATCH 37/57] Fix strip error when cross compiling. --- GNUmakefile | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index caab77e..54c88f0 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -147,6 +147,9 @@ CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ LDFLAGS = $(CURL_LIBS) $(LIBXML2_LIBS) $(OPENSSL_LIBS) -lpthread +STRIP ?= strip +INSTALL := install --strip-program=$(STRIP) + # -------------------------------------------------------------------------- # Default targets are everything @@ -168,11 +171,11 @@ exported: libs3 s3 headers .PHONY: install install: exported $(QUIET_ECHO) $(DESTDIR)/bin/s3: Installing executable - $(VERBOSE_SHOW) install -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rwx,go+rx $(BUILD)/bin/s3 \ $(DESTDIR)/bin/s3 $(QUIET_ECHO) \ $(LIBDIR)/libs3.so.$(LIBS3_VER): Installing shared library - $(VERBOSE_SHOW) install -Dps -m u+rw,go+r \ + $(VERBOSE_SHOW) $(INSTALL) -Dps -m u+rw,go+r \ $(BUILD)/lib/libs3.so.$(LIBS3_VER_MAJOR) \ $(LIBDIR)/libs3.so.$(LIBS3_VER) $(QUIET_ECHO) \ @@ -182,10 +185,10 @@ install: exported $(QUIET_ECHO) $(LIBDIR)/libs3.so: Linking shared library $(VERBOSE_SHOW) ln -sf libs3.so.$(LIBS3_VER_MAJOR) $(LIBDIR)/libs3.so $(QUIET_ECHO) $(LIBDIR)/libs3.a: Installing static library - $(VERBOSE_SHOW) install -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/lib/libs3.a \ $(LIBDIR)/libs3.a $(QUIET_ECHO) $(DESTDIR)/include/libs3.h: Installing header - $(VERBOSE_SHOW) install -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ + $(VERBOSE_SHOW) $(INSTALL) -Dp -m u+rw,go+r $(BUILD)/include/libs3.h \ $(DESTDIR)/include/libs3.h From 3d2f18ed11776a4b1c81041b6f22ceaf2f810ec7 Mon Sep 17 00:00:00 2001 From: Antonio Nesic Date: Tue, 4 Apr 2017 15:42:09 +0200 Subject: [PATCH 38/57] Minor fix for building on macOS Sierra --- GNUmakefile.osx | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GNUmakefile.osx b/GNUmakefile.osx index a7de08f..e40e0a5 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -125,7 +125,12 @@ ifndef CFLAGS endif endif -CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ +# -------------------------------------------------------------------------- +# -Werror breaks the build on the macOS Sierra rendering build unusable +# so we use -Wunused-parameter flag instead that will only issue a warning +# with the newest clang compiler + +CFLAGS += -Wall -Wunused-parameter -Wshadow -Wextra -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ From 5fbb16ee268036ea3e23c414a908b6633bf55c05 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Fri, 7 Apr 2017 12:26:26 +0300 Subject: [PATCH 39/57] create bucket: ignore response body --- GNUmakefile | 4 +++- src/bucket.c | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index caab77e..90eefeb 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -136,7 +136,9 @@ ifndef CFLAGS endif endif -CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ +CFLAGS += -Wall -Werror -Wshadow -Wextra \ + -Wno-unused-parameter \ + -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ diff --git a/src/bucket.c b/src/bucket.c index 0f852bc..71bc134 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -227,6 +227,13 @@ static void createBucketCompleteCallback(S3Status requestStatus, free(cbData); } +static S3Status createBucketFromS3Callback(int bufferSize, const char *buffer, + void *callbackData) +{ + // Sometimes S3 sends response body. We sillently ignore it. + return S3StatusOK; +} + void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, const char *secretAccessKey, const char *securityToken, const char *hostName, const char *bucketName, @@ -299,7 +306,7 @@ void S3_create_bucket(S3Protocol protocol, const char *accessKeyId, &createBucketPropertiesCallback, // propertiesCallback &createBucketDataCallback, // toS3Callback cbData->docLen, // toS3CallbackTotalSize - 0, // fromS3Callback + createBucketFromS3Callback, // fromS3Callback &createBucketCompleteCallback, // completeCallback cbData, // callbackData timeoutMs // timeoutMs From 84781bf1ab82d9940fc8090892a1a3cb8227467b Mon Sep 17 00:00:00 2001 From: Yury Samkevich Date: Thu, 11 May 2017 16:17:09 +0300 Subject: [PATCH 40/57] Added additional cheph specific error codes --- inc/libs3.h | 4 +++- src/error_parser.c | 4 +++- src/general.c | 4 +++- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index f48eaa8..aab820e 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -326,6 +326,7 @@ typedef enum S3StatusErrorInvalidBucketName , S3StatusErrorInvalidBucketState , S3StatusErrorInvalidDigest , + S3StatusErrorInvalidEncryptionAlgorithmError , S3StatusErrorInvalidLocationConstraint , S3StatusErrorInvalidObjectState , S3StatusErrorInvalidPart , @@ -361,7 +362,7 @@ typedef enum S3StatusErrorNoSuchVersion , S3StatusErrorNotImplemented , S3StatusErrorNotSignedUp , - S3StatusErrorNotSuchBucketPolicy , + S3StatusErrorNoSuchBucketPolicy , S3StatusErrorOperationAborted , S3StatusErrorPermanentRedirect , S3StatusErrorPreconditionFailed , @@ -380,6 +381,7 @@ typedef enum S3StatusErrorUnexpectedContent , S3StatusErrorUnresolvableGrantByEmailAddress , S3StatusErrorUserKeyMustBeSpecified , + S3StatusErrorQuotaExceeded , S3StatusErrorUnknown , /** diff --git a/src/error_parser.c b/src/error_parser.c index 7b4aedf..6c557ba 100644 --- a/src/error_parser.c +++ b/src/error_parser.c @@ -182,6 +182,7 @@ void error_parser_convert_status(ErrorParser *errorParser, S3Status *status) HANDLE_CODE(InvalidBucketName); HANDLE_CODE(InvalidBucketState); HANDLE_CODE(InvalidDigest); + HANDLE_CODE(InvalidEncryptionAlgorithmError); HANDLE_CODE(InvalidLocationConstraint); HANDLE_CODE(InvalidObjectState); HANDLE_CODE(InvalidPart); @@ -217,7 +218,7 @@ void error_parser_convert_status(ErrorParser *errorParser, S3Status *status) HANDLE_CODE(NoSuchVersion); HANDLE_CODE(NotImplemented); HANDLE_CODE(NotSignedUp); - HANDLE_CODE(NotSuchBucketPolicy); + HANDLE_CODE(NoSuchBucketPolicy); HANDLE_CODE(OperationAborted); HANDLE_CODE(PermanentRedirect); HANDLE_CODE(PreconditionFailed); @@ -236,6 +237,7 @@ void error_parser_convert_status(ErrorParser *errorParser, S3Status *status) HANDLE_CODE(UnexpectedContent); HANDLE_CODE(UnresolvableGrantByEmailAddress); HANDLE_CODE(UserKeyMustBeSpecified); + HANDLE_CODE(QuotaExceeded); *status = S3StatusErrorUnknown; code_set: diff --git a/src/general.c b/src/general.c index df13225..c31c492 100644 --- a/src/general.c +++ b/src/general.c @@ -131,6 +131,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorInvalidBucketName); handlecase(ErrorInvalidBucketState); handlecase(ErrorInvalidDigest); + handlecase(ErrorInvalidEncryptionAlgorithmError); handlecase(ErrorInvalidLocationConstraint); handlecase(ErrorInvalidObjectState); handlecase(ErrorInvalidPart); @@ -166,7 +167,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorNoSuchVersion); handlecase(ErrorNotImplemented); handlecase(ErrorNotSignedUp); - handlecase(ErrorNotSuchBucketPolicy); + handlecase(ErrorNoSuchBucketPolicy); handlecase(ErrorOperationAborted); handlecase(ErrorPermanentRedirect); handlecase(ErrorPreconditionFailed); @@ -185,6 +186,7 @@ const char *S3_get_status_name(S3Status status) handlecase(ErrorUnexpectedContent); handlecase(ErrorUnresolvableGrantByEmailAddress); handlecase(ErrorUserKeyMustBeSpecified); + handlecase(ErrorQuotaExceeded); handlecase(ErrorUnknown); handlecase(HttpErrorMovedTemporarily); handlecase(HttpErrorBadRequest); From 1db09f5201bb5020d53e02bc18331d36195aa117 Mon Sep 17 00:00:00 2001 From: Bryan Ischo Date: Wed, 24 May 2017 11:23:48 -0700 Subject: [PATCH 41/57] Fix signature error when requesting with sub resource. Original change from "Like Ma" (likema on github). --- src/request.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/request.c b/src/request.c index e1aad97..dead6fc 100644 --- a/src/request.c +++ b/src/request.c @@ -893,7 +893,9 @@ static void canonicalize_query_string(const char *queryParams, append("&"); } append(subResource); - append("="); + if (!strchr(subResource, '=')) { + append("="); + } } #undef append From bd4920b3a7ce79da5815ad3a4d981529886edb21 Mon Sep 17 00:00:00 2001 From: Like Ma Date: Wed, 24 May 2017 04:32:26 +0800 Subject: [PATCH 42/57] Fix urlencode error according to AWS S3 documents. --- inc/util.h | 2 +- src/bucket.c | 2 +- src/multipart.c | 4 ++-- src/request.c | 25 +++---------------------- src/util.c | 15 ++++++++++----- 5 files changed, 17 insertions(+), 31 deletions(-) diff --git a/inc/util.h b/inc/util.h index 9a65229..ba1f13e 100644 --- a/inc/util.h +++ b/inc/util.h @@ -76,7 +76,7 @@ // 3x the number of characters that [source] has. At most [maxSrcSize] bytes // from [src] are encoded; if more are present in [src], 0 is returned from // urlEncode, else nonzero is returned. -int urlEncode(char *dest, const char *src, int maxSrcSize); +int urlEncode(char *dest, const char *src, int maxSrcSize, int encodeSlash); // Returns < 0 on failure >= 0 on success int64_t parseIso8601Time(const char *str); diff --git a/src/bucket.c b/src/bucket.c index 0f852bc..bbfc428 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -674,7 +674,7 @@ void S3_list_bucket(const S3BucketContext *bucketContext, const char *prefix, } \ amp = 1; \ char encoded[3 * 1024]; \ - if (!urlEncode(encoded, value, 1024)) { \ + if (!urlEncode(encoded, value, 1024, 1)) { \ (*(handler->responseHandler.completeCallback)) \ (S3StatusQueryParamsTooLong, 0, callbackData); \ return; \ diff --git a/src/multipart.c b/src/multipart.c index 8b2fd32..70e59f3 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -872,7 +872,7 @@ void S3_list_multipart_uploads(S3BucketContext *bucketContext, } \ amp = 1; \ char encoded[3 * 1024]; \ - if (!urlEncode(encoded, value, 1024)) { \ + if (!urlEncode(encoded, value, 1024, 1)) { \ (*(handler->responseHandler.completeCallback)) \ (S3StatusQueryParamsTooLong, 0, callbackData); \ return; \ @@ -1000,7 +1000,7 @@ void S3_list_parts(S3BucketContext *bucketContext, const char *key, } \ amp = 1; \ char encoded[3 * 1024]; \ - if (!urlEncode(encoded, value, 1024)) { \ + if (!urlEncode(encoded, value, 1024, 1)) { \ (*(handler->responseHandler.completeCallback)) \ (S3StatusQueryParamsTooLong, 0, callbackData); \ return; \ diff --git a/src/request.c b/src/request.c index dead6fc..be162fb 100644 --- a/src/request.c +++ b/src/request.c @@ -629,7 +629,7 @@ static S3Status compose_standard_headers(const RequestParams *params, static S3Status encode_key(const RequestParams *params, RequestComputedValues *values) { - return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE) ? + return (urlEncode(values->urlEncodedKey, params->key, S3_MAX_KEY_SIZE, 0) ? S3StatusOK : S3StatusUriTooLong); } @@ -843,28 +843,9 @@ static void sort_and_urlencode_query_string(const char *queryString, #endif unsigned int pi = 0; - char appendage[4]; - for (; pi < numParams; pi++) { - const char *param = params[pi]; - int foundEquals = 0; - for (i = 0; i < strlen(param); i++) { - char c = param[i]; - if (isalnum(c) || (c == '_') || (c == '-') || (c == '~') - || (c == '.')) { - appendage[0] = c; - appendage[1] = '\0'; - } - else if ((c == '=') && !foundEquals) { - appendage[0] = c; - appendage[1] = '\0'; - foundEquals = 1; - } - else { - snprintf(appendage, 4, "%%%02X", c); - } - strncat(result, appendage, strlen(appendage)); - } + // All params are urlEncoded + strncat(result, params[pi], strlen(params[pi])); strncat(result, "&", 1); } result[strlen(result) - 1] = '\0'; diff --git a/src/util.c b/src/util.c index bb2ca97..9c35934 100644 --- a/src/util.c +++ b/src/util.c @@ -53,8 +53,14 @@ static int checkString(const char *str, const char *format) return 1; } - -int urlEncode(char *dest, const char *src, int maxSrcSize) +/* + * Encode rules: + * 1. Every byte except: 'A'-'Z', 'a'-'z', '0'-'9', '-', '.', '_', and '~' + * 2. The space must be encoded as "%20" (and not as "+") + * 3. Letters in the hexadecimal value must be uppercase, for example "%1A" + * 4. Encode the forward slash character, '/', everywhere except in the object key name + */ +int urlEncode(char *dest, const char *src, int maxSrcSize, int encodeSlash) { static const char *hex = "0123456789ABCDEF"; @@ -67,9 +73,8 @@ int urlEncode(char *dest, const char *src, int maxSrcSize) } unsigned char c = *src; if (isalnum(c) || - (c == '-') || (c == '_') || (c == '.') || (c == '!') || - (c == '~') || (c == '*') || (c == '\'') || (c == '(') || - (c == ')') || (c == '/')) { + (c == '-') || (c == '_') || (c == '.') || + (c == '~') || (c == '/' && !encodeSlash)) { *dest++ = c; } else { From 728137b557d51940f450766892f7bb54d211b832 Mon Sep 17 00:00:00 2001 From: Like Ma Date: Fri, 26 May 2017 16:08:50 +0800 Subject: [PATCH 43/57] Rename sort_and_urlencode_query_string to sort_query_string. --- src/request.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/request.c b/src/request.c index be162fb..2de71a3 100644 --- a/src/request.c +++ b/src/request.c @@ -805,8 +805,7 @@ static void canonicalize_resource(const S3BucketContext *context, } -static void sort_and_urlencode_query_string(const char *queryString, - char *result) +static void sort_query_string(const char *queryString, char *result) { #ifdef SIGNATURE_DEBUG printf("\n--\nsort_and_urlencode\nqueryString: %s\n", queryString); @@ -865,7 +864,7 @@ static void canonicalize_query_string(const char *queryParams, if (queryParams && queryParams[0]) { char sorted[strlen(queryParams) * 2]; sorted[0] = '\0'; - sort_and_urlencode_query_string(queryParams, sorted); + sort_query_string(queryParams, sorted); append(sorted); } From adaaeccb5f0b3424ba8d428b5519654794d8e0d4 Mon Sep 17 00:00:00 2001 From: Andrei Kopats Date: Wed, 31 May 2017 17:41:22 +0300 Subject: [PATCH 44/57] remove -Wunused-parameter --- GNUmakefile | 1 - src/bucket.c | 5 +++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 90eefeb..bcb909f 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -137,7 +137,6 @@ ifndef CFLAGS endif CFLAGS += -Wall -Werror -Wshadow -Wextra \ - -Wno-unused-parameter \ -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ diff --git a/src/bucket.c b/src/bucket.c index 71bc134..1b6b24e 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -231,6 +231,11 @@ static S3Status createBucketFromS3Callback(int bufferSize, const char *buffer, void *callbackData) { // Sometimes S3 sends response body. We sillently ignore it. + + (void)bufferSize; // avoid unused parameter warning + (void)buffer; // avoid unused parameter warning + (void)callbackData; // avoid unused parameter warning + return S3StatusOK; } From ce36482e32a65cec97c31bad92ff7425d49c7293 Mon Sep 17 00:00:00 2001 From: Like Ma Date: Wed, 15 Mar 2017 15:53:17 +0800 Subject: [PATCH 45/57] Fix crash in Windows by init and deinit libxml2 parser. --- src/request.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/request.c b/src/request.c index 2de71a3..63174d0 100644 --- a/src/request.c +++ b/src/request.c @@ -29,6 +29,7 @@ #include #include #include +#include #include "request.h" #include "request_context.h" #include "response_headers_handler.h" @@ -1452,6 +1453,7 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, "Mozilla/4.0 (Compatible; %s; libs3 %s.%s; %s)", userAgentInfo, LIBS3_VER_MAJOR, LIBS3_VER_MINOR, platform); + xmlInitParser(); return S3StatusOK; } @@ -1460,6 +1462,7 @@ void request_api_deinitialize() { pthread_mutex_destroy(&requestStackMutexG); + xmlCleanupParser(); while (requestStackCountG--) { request_destroy(requestStackG[requestStackCountG]); } From 05deb102556cd8a06da0940f90f7d22e4f61f162 Mon Sep 17 00:00:00 2001 From: "sergey.dobrodey" Date: Mon, 2 Oct 2017 13:13:32 +0300 Subject: [PATCH 46/57] fixes https://github.com/bji/libs3/issues/21 --- src/response_headers_handler.c | 3 ++- src/s3.c | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/response_headers_handler.c b/src/response_headers_handler.c index 70046bc..bc4c175 100644 --- a/src/response_headers_handler.c +++ b/src/response_headers_handler.c @@ -145,7 +145,8 @@ void response_headers_handler_add(ResponseHeadersHandler *handler, string_multibuffer_add(handler->responsePropertyStrings, c, valuelen, fit); } - else if (!strncasecmp(header, "ETag", namelen)) { + else if ((!strncasecmp(header, "ETag", namelen)) + || (!strncasecmp(header, "Etag", namelen))) { // some servers reply with Etag header responseProperties->eTag = string_multibuffer_current(handler->responsePropertyStrings); string_multibuffer_add(handler->responsePropertyStrings, c, diff --git a/src/s3.c b/src/s3.c index b8d4405..88d3ddb 100644 --- a/src/s3.c +++ b/src/s3.c @@ -489,6 +489,7 @@ static void growbuffer_read(growbuffer **gb, int amt, int *amtReturn, buf->next->prev = buf->prev; } free(buf); + buf = NULL; } } @@ -2436,6 +2437,7 @@ static void put_object(int argc, char **argv, int optindex, MULTIPART_CHUNK_SIZE); MultipartPartData partData; + memset(&partData, 0, sizeof(MultipartPartData)); int partContentLength = 0; S3MultipartInitialHandler handler = { @@ -2486,10 +2488,11 @@ static void put_object(int argc, char **argv, int optindex, upload: todoContentLength -= MULTIPART_CHUNK_SIZE * manager.next_etags_pos; for (seq = manager.next_etags_pos + 1; seq <= totalSeq; seq++) { - memset(&partData, 0, sizeof(MultipartPartData)); partData.manager = &manager; partData.seq = seq; - partData.put_object_data = data; + if (partData.put_object_data.gb==NULL) { + partData.put_object_data = data; + } partContentLength = ((contentLength > MULTIPART_CHUNK_SIZE) ? MULTIPART_CHUNK_SIZE : contentLength); printf("%s Part Seq %d, length=%d\n", srcSize ? "Copying" : "Sending", seq, partContentLength); From e6ebeb70fc4263ab66caba4e00bdff9da278c5f0 Mon Sep 17 00:00:00 2001 From: Alexei Potashnik Date: Wed, 18 Oct 2017 10:10:59 -0700 Subject: [PATCH 47/57] extension to support curl_multi_socket_action operation Make libs3 work with curl_multi_socket_action mode where libs3 user will drive socket polling instructed directly by curl through a set of callbacks. This is an addition to the curl_multi_perform mode where libs3 does the polling itself. --- inc/libs3.h | 87 ++++++++++++++++++++++++++ inc/request_context.h | 12 ++++ src/request.c | 12 +++- src/request_context.c | 139 ++++++++++++++++++++++++++++-------------- 4 files changed, 202 insertions(+), 48 deletions(-) diff --git a/inc/libs3.h b/inc/libs3.h index aab820e..47cc19b 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -1240,6 +1240,28 @@ typedef S3Status (S3MultipartCommitResponseCallback)(const char *location, void *callbackData); +/** + * Mechanism for S3 application to customize each CURL easy request + * associated with the given S3 request context. + * + * This callback can be optinally configured using S3_create_request_context_ex + * and will be invoked every time a new CURL request is created in the + * context of the given CURLM handle. Invocation will occur after + * libs3 has finished configuring its own options of CURL, but before + * CURL is started. + * + * @param curl_multi is the CURLM handle associated with this context. + * @param curl_easy is the CURL request being created. + * @param setupData is the setupCurlCallbackData parameter passed to + * S3_create_request_context_ex. + * @return S3StatusOK to continue processing the request, anything else to + * immediately abort the request and pass this status + * to the S3ResponseCompleteCallback for this request. + **/ +typedef S3Status (*S3SetupCurlCallback)(void *curlMulti, void *curlEasy, + void *setupData); + + /** ************************************************************************** * Callback Structures ************************************************************************** **/ @@ -1583,6 +1605,51 @@ int S3_status_is_retryable(S3Status status); S3Status S3_create_request_context(S3RequestContext **requestContextReturn); +/** + * Extended version of S3_create_request_context used to create S3RequestContext + * for curl_multi_socket_action CURLM handles that will be managed by libs3 user. + * This type of handles offer better performance for applications with large + * number of simultaneous connections. For details, see MULTI_SOCKET chapter here: + * https://curl.haxx.se/libcurl/c/libcurl-multi.html + * + * In this mode libs3 user will + * - create its own CURLM using curl_multi_init() + * - configure it for its own handlers using + * CURLMOPT_SOCKETFUNCTION/CURLMOPT_TIMERFUNCTION/etc + * - use S3_create_request_context_ex to create S3RequestContext + * for the above CURLM handle + * - start S3 request + * - every time setupCurlCallback is called, will configure new CURL + * object with its own handlers using + * CURLOPT_OPENSOCKETFUNCTION/CURLOPT_CLOSESOCKETFUNCTION/etc + * - the moment libs3 adds CURL object to CURLM handle, curl will start + * communicating directly with libs3 user to drive socket operations, + * where libs3 user will be responsible for calling curl_multi_socket_action + * when necessary. + * - whenever curl_multi_socket_action indicates change in running_handles + * libs3 user should call S3_process_request_context to let libs3 process + * any completed curl transfers and notify back to libs3 user if that was + * the final transfer for a given S3 request. + * + * @param requestContextReturn returns the newly-created S3RequestContext + * structure, which if successfully returned, must be destroyed via a + * call to S3_destroy_request_context when it is no longer needed. If + * an error status is returned from this function, then + * requestContextReturn will not have been filled in, and + * S3_destroy_request_context should not be called on it + * @param curlMulti is the CURLM handle to be associated with this context. + * @param setupCurlCallback is an optional callback routine to be invoked + * by libs3 every time another CURL request is being created for + * use in this context. + * @param setupCurlCallbackData is an opaque data to be passed to + * setupCurlCallback. + **/ +S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn, + void *curlMulti, + S3SetupCurlCallback setupCurlCallback, + void *setupCurlCallbackData); + + /** * Destroys an S3RequestContext which was created with * S3_create_request_context. Any requests which are currently being @@ -1640,6 +1707,26 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext, int *requestsRemainingReturn); +/** + * Extract and finish requests completed by curl multi handle mechanism + * in curl_multi_socket_action mode. Should be called by libs3 user when + * curl_multi_socket_action indicates a change in running_handles. + * + * @param requestContext is the S3RequestContext to process + * @return One of: + * S3StatusOK if request processing proceeded without error + * S3StatusConnectionFailed if the socket connection to the server + * failed + * S3StatusServerFailedVerification if the SSL certificate of the + * server could not be verified. + * S3StatusInternalError if an internal error prevented the + * S3RequestContext from running one or more requests + * S3StatusOutOfMemory if requests could not be processed due to + * an out of memory error + **/ +S3Status S3_process_request_context(S3RequestContext *requestContext); + + /** * This function, in conjunction allows callers to manually manage a set of * requests using an S3RequestContext. This function returns the set of file diff --git a/inc/request_context.h b/inc/request_context.h index 229ef35..b697cc0 100644 --- a/inc/request_context.h +++ b/inc/request_context.h @@ -29,14 +29,26 @@ #include "libs3.h" + +typedef enum +{ + S3CurlModeMultiPerform , + S3CurlModeMultiSocket , +} S3CurlMode; + + struct S3RequestContext { CURLM *curlm; + S3CurlMode curl_mode; int verifyPeerSet; long verifyPeer; struct Request *requests; + + S3SetupCurlCallback setupCurlCallback; + void *setupCurlCallbackData; }; diff --git a/src/request.c b/src/request.c index 63174d0..156ea32 100644 --- a/src/request.c +++ b/src/request.c @@ -1301,6 +1301,7 @@ static void request_deinitialize(Request *request) static S3Status request_get(const RequestParams *params, const RequestComputedValues *values, + const S3RequestContext *context, Request **reqReturn) { Request *request = 0; @@ -1360,6 +1361,15 @@ static S3Status request_get(const RequestParams *params, return status; } + if (context && context->setupCurlCallback && + (status = context->setupCurlCallback( + context->curlm, request->curl, + context->setupCurlCallbackData)) != S3StatusOK) { + curl_easy_cleanup(request->curl); + free(request); + return status; + } + request->propertiesCallback = params->propertiesCallback; request->toS3Callback = params->toS3Callback; @@ -1548,7 +1558,7 @@ void request_perform(const RequestParams *params, S3RequestContext *context) } // Get an initialized Request structure now - if ((status = request_get(params, &computed, &request)) != S3StatusOK) { + if ((status = request_get(params, &computed, context, &request)) != S3StatusOK) { return_status(status); } if (context && context->verifyPeerSet) { diff --git a/src/request_context.c b/src/request_context.c index cf82f2d..eeecbd6 100644 --- a/src/request_context.c +++ b/src/request_context.c @@ -31,7 +31,10 @@ #include "request_context.h" -S3Status S3_create_request_context(S3RequestContext **requestContextReturn) +S3Status S3_create_request_context_ex(S3RequestContext **requestContextReturn, + CURLM *curlm, + S3SetupCurlCallback setupCurlCallback, + void *setupCurlCallbackData) { *requestContextReturn = (S3RequestContext *) malloc(sizeof(S3RequestContext)); @@ -40,19 +43,35 @@ S3Status S3_create_request_context(S3RequestContext **requestContextReturn) return S3StatusOutOfMemory; } - if (!((*requestContextReturn)->curlm = curl_multi_init())) { - free(*requestContextReturn); - return S3StatusOutOfMemory; + if (curlm) { + (*requestContextReturn)->curlm = curlm; + (*requestContextReturn)->curl_mode = S3CurlModeMultiSocket; + } + else { + if (!((*requestContextReturn)->curlm = curl_multi_init())) { + free(*requestContextReturn); + return S3StatusOutOfMemory; + } + + (*requestContextReturn)->curl_mode = S3CurlModeMultiPerform; } (*requestContextReturn)->requests = 0; (*requestContextReturn)->verifyPeer = 0; (*requestContextReturn)->verifyPeerSet = 0; + (*requestContextReturn)->setupCurlCallback = setupCurlCallback; + (*requestContextReturn)->setupCurlCallbackData = setupCurlCallbackData; return S3StatusOK; } +S3Status S3_create_request_context(S3RequestContext **requestContextReturn) +{ + return S3_create_request_context_ex(requestContextReturn, NULL, NULL, NULL); +} + + void S3_destroy_request_context(S3RequestContext *requestContext) { // For each request in the context, remove curl handle, call back its done @@ -68,7 +87,8 @@ void S3_destroy_request_context(S3RequestContext *requestContext) r = rNext; } while (r != rFirst); - curl_multi_cleanup(requestContext->curlm); + if (requestContext->curl_mode == S3CurlModeMultiPerform) + curl_multi_cleanup(requestContext->curlm); free(requestContext); } @@ -109,10 +129,62 @@ S3Status S3_runall_request_context(S3RequestContext *requestContext) } +static S3Status process_request_context(S3RequestContext *requestContext, int *retry) +{ + CURLMsg *msg; + int junk; + + *retry = 0; + + while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) { + if (msg->msg != CURLMSG_DONE) { + return S3StatusInternalError; + } + Request *request; + if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, + (char **) (char *) &request) != CURLE_OK) { + return S3StatusInternalError; + } + // Remove the request from the list of requests + if (request->prev == request->next) { + // It was the only one on the list + requestContext->requests = 0; + } + else { + // It doesn't matter what the order of them are, so just in + // case request was at the head of the list, put the one after + // request to the head of the list + requestContext->requests = request->next; + request->prev->next = request->next; + request->next->prev = request->prev; + } + if ((msg->data.result != CURLE_OK) && + (request->status == S3StatusOK)) { + request->status = request_curl_code_to_status( + msg->data.result); + } + if (curl_multi_remove_handle(requestContext->curlm, + msg->easy_handle) != CURLM_OK) { + return S3StatusInternalError; + } + // Finish the request, ensuring that all callbacks have been made, + // and also releases the request + request_finish(request); + // Now, since a callback was made, there may be new requests + // queued up to be performed immediately, so do so + *retry = 1; + } + + return S3StatusOK; +} + + S3Status S3_runonce_request_context(S3RequestContext *requestContext, int *requestsRemainingReturn) { + S3Status s3_status; CURLMcode status; + int retry; do { status = curl_multi_perform(requestContext->curlm, @@ -128,51 +200,24 @@ S3Status S3_runonce_request_context(S3RequestContext *requestContext, return S3StatusInternalError; } - CURLMsg *msg; - int junk; - while ((msg = curl_multi_info_read(requestContext->curlm, &junk))) { - if (msg->msg != CURLMSG_DONE) { - return S3StatusInternalError; - } - Request *request; - if (curl_easy_getinfo(msg->easy_handle, CURLINFO_PRIVATE, - (char **) (char *) &request) != CURLE_OK) { - return S3StatusInternalError; - } - // Remove the request from the list of requests - if (request->prev == request->next) { - // It was the only one on the list - requestContext->requests = 0; - } - else { - // It doesn't matter what the order of them are, so just in - // case request was at the head of the list, put the one after - // request to the head of the list - requestContext->requests = request->next; - request->prev->next = request->next; - request->next->prev = request->prev; - } - if ((msg->data.result != CURLE_OK) && - (request->status == S3StatusOK)) { - request->status = request_curl_code_to_status - (msg->data.result); - } - if (curl_multi_remove_handle(requestContext->curlm, - msg->easy_handle) != CURLM_OK) { - return S3StatusInternalError; - } - // Finish the request, ensuring that all callbacks have been made, - // and also releases the request - request_finish(request); - // Now, since a callback was made, there may be new requests - // queued up to be performed immediately, so do so - status = CURLM_CALL_MULTI_PERFORM; - } - } while (status == CURLM_CALL_MULTI_PERFORM); + s3_status = process_request_context(requestContext, &retry); + } while (s3_status == S3StatusOK && + (status == CURLM_CALL_MULTI_PERFORM || retry)); - return S3StatusOK; + return s3_status; } + +S3Status S3_process_request_context(S3RequestContext *requestContext) +{ + int retry; + /* In curl_multi_socket_action mode any new requests created during + the following call will have already started associated socket + operations, so no need to retry here */ + return process_request_context(requestContext, &retry); +} + + S3Status S3_get_request_context_fdsets(S3RequestContext *requestContext, fd_set *readFdSet, fd_set *writeFdSet, fd_set *exceptFdSet, int *maxFd) From 34340ec633e1c94a215390c11dedbef439d540cb Mon Sep 17 00:00:00 2001 From: yuliyang Date: Tue, 3 Jul 2018 15:16:41 +0800 Subject: [PATCH 48/57] add bucket-owner-full-control Signed-off-by: yuliyang --- inc/libs3.h | 3 ++- src/request.c | 3 +++ src/s3.c | 3 +++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/inc/libs3.h b/inc/libs3.h index aab820e..447f338 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -498,7 +498,8 @@ typedef enum S3CannedAclPrivate = 0, /* private */ S3CannedAclPublicRead = 1, /* public-read */ S3CannedAclPublicReadWrite = 2, /* public-read-write */ - S3CannedAclAuthenticatedRead = 3 /* authenticated-read */ + S3CannedAclAuthenticatedRead = 3, /* authenticated-read */ + S3CannedAclBucketOwnerFullControl = 4 /* bucket-owner-full-control */ } S3CannedAcl; diff --git a/src/request.c b/src/request.c index 63174d0..50d3ca8 100644 --- a/src/request.c +++ b/src/request.c @@ -361,6 +361,9 @@ static S3Status compose_amz_headers(const RequestParams *params, case S3CannedAclPublicReadWrite: cannedAclString = "public-read-write"; break; + case S3CannedAclBucketOwnerFullControl: + cannedAclString = "bucket-owner-full-control"; + break; default: // S3CannedAclAuthenticatedRead cannedAclString = "authenticated-read"; break; diff --git a/src/s3.c b/src/s3.c index b8d4405..5eed317 100644 --- a/src/s3.c +++ b/src/s3.c @@ -2282,6 +2282,9 @@ static void put_object(int argc, char **argv, int optindex, else if (!strcmp(val, "public-read-write")) { cannedAcl = S3CannedAclPublicReadWrite; } + else if (!strcmp(val, "bucket-owner-full-control")) { + cannedAcl = S3CannedAclBucketOwnerFullControl; + } else if (!strcmp(val, "authenticated-read")) { cannedAcl = S3CannedAclAuthenticatedRead; } From f9c5fb8120e6f9fdd6578b8c075984611dbebf01 Mon Sep 17 00:00:00 2001 From: Claude Bing Date: Tue, 21 Aug 2018 12:22:27 -0400 Subject: [PATCH 49/57] Increase maximum object size Following AWS S3's increased single-object size limit, allow the upload of data > 5 GB --- src/s3.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/s3.c b/src/s3.c index b8d4405..45c733c 100644 --- a/src/s3.c +++ b/src/s3.c @@ -2201,9 +2201,9 @@ static void put_object(int argc, char **argv, int optindex, CONTENT_LENGTH_PREFIX_LEN)) { contentLength = convertInt(&(param[CONTENT_LENGTH_PREFIX_LEN]), "contentLength"); - if (contentLength > (5LL * 1024 * 1024 * 1024)) { + if (contentLength > (5LL * 1024 * 1024 * 1024 * 1024)) { fprintf(stderr, "\nERROR: contentLength must be no greater " - "than 5 GB\n"); + "than 5 TB\n"); usageExit(stderr); } } From 691372307831cec3269ed31e48f614eab2f65c7d Mon Sep 17 00:00:00 2001 From: Claude Bing Date: Tue, 21 Aug 2018 12:27:13 -0400 Subject: [PATCH 50/57] Fix growbuffer_append This fixes the following error encountered when uploading large files: ERROR: ErrorMalformedXML Message: The XML you provided was not well-formed or did not validate against our published schema. This becomes a problem after a previous patch increases the size limit for a single object, causing more than one growbuffer to be allocated for holding the XML. --- src/s3.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/s3.c b/src/s3.c index 45c733c..2ac76eb 100644 --- a/src/s3.c +++ b/src/s3.c @@ -426,7 +426,7 @@ typedef struct growbuffer // returns nonzero on success, zero on out of memory static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) { - int toCopy = 0 ; + int origDataLen = dataLen; while (dataLen) { growbuffer *buf = *gb ? (*gb)->prev : 0; if (!buf || (buf->size == sizeof(buf->data))) { @@ -448,7 +448,7 @@ static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) } } - toCopy = (sizeof(buf->data) - buf->size); + int toCopy = (sizeof(buf->data) - buf->size); if (toCopy > dataLen) { toCopy = dataLen; } @@ -458,7 +458,7 @@ static int growbuffer_append(growbuffer **gb, const char *data, int dataLen) buf->size += toCopy, data += toCopy, dataLen -= toCopy; } - return toCopy; + return origDataLen; } From 7676aa93ab59cdd233fb27074741eb8d0dc18a87 Mon Sep 17 00:00:00 2001 From: Claude Bing Date: Mon, 10 Sep 2018 14:14:55 -0400 Subject: [PATCH 51/57] Increase MULTIPART_CHUNK_SIZE Increasing the chunk size avoids an issue where too many parts are uploaded, triggering an error when calling the S3 CompleteMultipartUpload API command. The current limit on the number of parts per upload is 10,000 according to the S3 documentation (https://docs.aws.amazon.com/AmazonS3/latest/dev/qfacts.html) --- src/s3.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/s3.c b/src/s3.c index 2ac76eb..edb71c4 100644 --- a/src/s3.c +++ b/src/s3.c @@ -2057,7 +2057,7 @@ static int putObjectDataCallback(int bufferSize, char *buffer, return ret; } -#define MULTIPART_CHUNK_SIZE (15 << 20) // multipart is 15M +#define MULTIPART_CHUNK_SIZE (768 << 20) // multipart is 768M typedef struct MultipartPartData { put_object_callback_data put_object_data; From 795ba3f589608f47d86e286b646231aa08cae8e8 Mon Sep 17 00:00:00 2001 From: ffan Date: Thu, 27 Sep 2018 14:15:04 +0800 Subject: [PATCH 52/57] fix bug on S3_initiate_multipart S3_initiate_multipar will save param callbackData in struct InitialMultipartData, and passed InitialMultipartData* as callback data to S3PropertiesCallback, which make a crash when extract user defined data from S3PropertiesCallback --- src/multipart.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/multipart.c b/src/multipart.c index 70e59f3..606ad4d 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -94,6 +94,14 @@ static S3Status initialMultipartXmlCallback(const char *elementPath, return S3StatusOK; } +static S3Status InitialMultipartPropertiesCallback(const S3ResponseProperties *properties, + void *callbackData) +{ + InitialMultipartData *mdata = (InitialMultipartData *)callbackData; + return mdata->handler->responseHandler.propertiesCallback(properties, mdata->userdata); +} + + void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, S3PutProperties *putProperties, S3MultipartInitialHandler *handler, @@ -129,7 +137,7 @@ void S3_initiate_multipart(S3BucketContext *bucketContext, const char *key, 0, // startByte 0, // byteCount putProperties, // putProperties - handler->responseHandler.propertiesCallback, // propertiesCallback + InitialMultipartPropertiesCallback, // propertiesCallback 0, // toS3Callback 0, // toS3CallbackTotalSize InitialMultipartCallback, // fromS3Callback From 537bb7b59bbf3d9eb322eb807d329bb701075a88 Mon Sep 17 00:00:00 2001 From: Bryan Ischo Date: Thu, 4 Oct 2018 12:21:15 -0700 Subject: [PATCH 53/57] Added GPLv2 as alternate license terms, and specified that the software is licensed under LGPLv3 *or above* or GPLv2 *or above* (user's choice). --- COPYING-GPLv2 | 339 +++++++++++++++++++++++++++++++++ COPYING => COPYING-LGPLv3 | 0 GNUmakefile | 8 +- GNUmakefile.mingw | 10 +- GNUmakefile.osx | 10 +- LICENSE | 12 +- inc/error_parser.h | 12 +- inc/libs3.h | 8 +- inc/mingw/pthread.h | 12 +- inc/mingw/sys/select.h | 12 +- inc/mingw/sys/utsname.h | 12 +- inc/request.h | 8 +- inc/request_context.h | 12 +- inc/response_headers_handler.h | 12 +- inc/simplexml.h | 12 +- inc/string_buffer.h | 12 +- inc/util.h | 12 +- src/bucket.c | 8 +- src/bucket_metadata.c | 8 +- src/error_parser.c | 12 +- src/general.c | 8 +- src/mingw_functions.c | 12 +- src/mingw_s3_functions.c | 12 +- src/multipart.c | 8 +- src/object.c | 8 +- src/request.c | 8 +- src/request_context.c | 12 +- src/response_headers_handler.c | 12 +- src/s3.c | 8 +- src/service.c | 8 +- src/service_access_logging.c | 8 +- src/simplexml.c | 12 +- src/testsimplexml.c | 12 +- src/util.c | 12 +- 34 files changed, 600 insertions(+), 71 deletions(-) create mode 100644 COPYING-GPLv2 rename COPYING => COPYING-LGPLv3 (100%) diff --git a/COPYING-GPLv2 b/COPYING-GPLv2 new file mode 100644 index 0000000..d159169 --- /dev/null +++ b/COPYING-GPLv2 @@ -0,0 +1,339 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Lesser General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. diff --git a/COPYING b/COPYING-LGPLv3 similarity index 100% rename from COPYING rename to COPYING-LGPLv3 diff --git a/GNUmakefile b/GNUmakefile index c81537c..8ab4109 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -6,7 +6,9 @@ # # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, version 3 of the License. +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. # # In addition, as a special exception, the copyright holders give # permission to link the code of this library and its programs with the @@ -20,6 +22,10 @@ # You should have received a copy of the GNU Lesser General Public License # version 3 along with libs3, in a file named COPYING. If not, see # . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . # I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools # but I just couldn't stomach them. Since this is a Makefile for POSIX diff --git a/GNUmakefile.mingw b/GNUmakefile.mingw index ba1ad57..bad5b78 100644 --- a/GNUmakefile.mingw +++ b/GNUmakefile.mingw @@ -1,12 +1,12 @@ # GNUmakefile.mingw # -# Copyright 2008 Bryan Ischo -# # This file is part of libs3. # # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, version 3 of the License. +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. # # In addition, as a special exception, the copyright holders give # permission to link the code of this library and its programs with the @@ -20,6 +20,10 @@ # You should have received a copy of the GNU Lesser General Public License # version 3 along with libs3, in a file named COPYING. If not, see # . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . # I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools # but I just couldn't stomach them. Since this is a Makefile for POSIX diff --git a/GNUmakefile.osx b/GNUmakefile.osx index e40e0a5..c56d1d7 100644 --- a/GNUmakefile.osx +++ b/GNUmakefile.osx @@ -1,12 +1,12 @@ # GNUmakefile.osx # -# Copyright 2008 Bryan Ischo -# # This file is part of libs3. # # libs3 is free software: you can redistribute it and/or modify it under the # terms of the GNU Lesser General Public License as published by the Free -# Software Foundation, version 3 of the License. +# Software Foundation, version 3 or above of the License. You can also +# redistribute and/or modify it under the terms of the GNU General Public +# License, version 2 or above of the License. # # In addition, as a special exception, the copyright holders give # permission to link the code of this library and its programs with the @@ -20,6 +20,10 @@ # You should have received a copy of the GNU Lesser General Public License # version 3 along with libs3, in a file named COPYING. If not, see # . +# +# You should also have received a copy of the GNU General Public License +# version 2 along with libs3, in a file named COPYING-GPLv2. If not, see +# . # I tried to use the autoconf/automake/autolocal/etc (i.e. autohell) tools # but I just couldn't stomach them. Since this is a Makefile for POSIX diff --git a/LICENSE b/LICENSE index adcbfa5..ecd700e 100644 --- a/LICENSE +++ b/LICENSE @@ -2,7 +2,11 @@ Copyright 2008 Bryan Ischo libs3 is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free -Software Foundation, version 3 of the License. +Software Foundation, version 3 or above of the License. + +You can also redistribute it and/or modify it under the terms of the +GNU General Public License as published by the Free Software Foundation, +version 2 or above of the License. In addition, as a special exception, the copyright holders give permission to link the code of this library and its programs with the @@ -14,5 +18,9 @@ FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU Lesser General Public License -version 3 along with libs3, in a file named COPYING. If not, see +version 3 along with libs3, in a file named COPYING-LGPLv3. If not, see +. + +You should also have received a copy of the GNU General Public License +version 2 along with libs3, in a file named COPYING-GPLv2. If not, see . diff --git a/inc/error_parser.h b/inc/error_parser.h index 3f946ac..86b2889 100644 --- a/inc/error_parser.h +++ b/inc/error_parser.h @@ -2,12 +2,14 @@ * error_parser.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef ERROR_PARSER_H diff --git a/inc/libs3.h b/inc/libs3.h index 5507abe..3318ef2 100644 --- a/inc/libs3.h +++ b/inc/libs3.h @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef LIBS3_H diff --git a/inc/mingw/pthread.h b/inc/mingw/pthread.h index 9664dd0..e87b580 100644 --- a/inc/mingw/pthread.h +++ b/inc/mingw/pthread.h @@ -2,12 +2,14 @@ * pthread.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef PTHREAD_H diff --git a/inc/mingw/sys/select.h b/inc/mingw/sys/select.h index c0b0484..3093717 100644 --- a/inc/mingw/sys/select.h +++ b/inc/mingw/sys/select.h @@ -2,12 +2,14 @@ * select.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ // This file is used only on a MingW build, and converts an include of diff --git a/inc/mingw/sys/utsname.h b/inc/mingw/sys/utsname.h index 8c351ee..44c4d22 100644 --- a/inc/mingw/sys/utsname.h +++ b/inc/mingw/sys/utsname.h @@ -2,12 +2,14 @@ * utsname.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ // This file is used only on a MingW build, and provides an implementation diff --git a/inc/request.h b/inc/request.h index 0143af7..a80971a 100644 --- a/inc/request.h +++ b/inc/request.h @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef REQUEST_H diff --git a/inc/request_context.h b/inc/request_context.h index b697cc0..e91d9c8 100644 --- a/inc/request_context.h +++ b/inc/request_context.h @@ -2,12 +2,14 @@ * request_context.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef REQUEST_CONTEXT_H diff --git a/inc/response_headers_handler.h b/inc/response_headers_handler.h index ba573e0..b2341c2 100644 --- a/inc/response_headers_handler.h +++ b/inc/response_headers_handler.h @@ -2,12 +2,14 @@ * response_headers_handler.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef RESPONSE_HEADERS_HANDLER_H diff --git a/inc/simplexml.h b/inc/simplexml.h index 4fbbf68..f12a22b 100644 --- a/inc/simplexml.h +++ b/inc/simplexml.h @@ -2,12 +2,14 @@ * simplexml.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef SIMPLEXML_H diff --git a/inc/string_buffer.h b/inc/string_buffer.h index d1cf5c3..a4968ad 100644 --- a/inc/string_buffer.h +++ b/inc/string_buffer.h @@ -2,12 +2,14 @@ * string_buffer.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef STRING_BUFFER_H diff --git a/inc/util.h b/inc/util.h index ba1f13e..8c66a55 100644 --- a/inc/util.h +++ b/inc/util.h @@ -2,12 +2,14 @@ * util.h * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #ifndef UTIL_H diff --git a/src/bucket.c b/src/bucket.c index c112e4d..6d30674 100644 --- a/src/bucket.c +++ b/src/bucket.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/bucket_metadata.c b/src/bucket_metadata.c index c2f1d0d..05f3e67 100644 --- a/src/bucket_metadata.c +++ b/src/bucket_metadata.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/error_parser.c b/src/error_parser.c index 6c557ba..e573aed 100644 --- a/src/error_parser.c +++ b/src/error_parser.c @@ -2,12 +2,14 @@ * error_parser.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/general.c b/src/general.c index c31c492..7876f58 100644 --- a/src/general.c +++ b/src/general.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/mingw_functions.c b/src/mingw_functions.c index e8295fe..421f757 100644 --- a/src/mingw_functions.c +++ b/src/mingw_functions.c @@ -2,12 +2,14 @@ * mingw_functions.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/mingw_s3_functions.c b/src/mingw_s3_functions.c index 675c65a..5098a61 100644 --- a/src/mingw_s3_functions.c +++ b/src/mingw_s3_functions.c @@ -2,12 +2,14 @@ * mingw_s3_functions.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ int setenv(const char *a, const char *b, int c) diff --git a/src/multipart.c b/src/multipart.c index 70e59f3..deaabbe 100644 --- a/src/multipart.c +++ b/src/multipart.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/object.c b/src/object.c index a852519..e8df4b0 100644 --- a/src/object.c +++ b/src/object.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/request.c b/src/request.c index 637e0cd..4d43cf9 100644 --- a/src/request.c +++ b/src/request.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/request_context.c b/src/request_context.c index eeecbd6..345096b 100644 --- a/src/request_context.c +++ b/src/request_context.c @@ -2,12 +2,14 @@ * request_context.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/response_headers_handler.c b/src/response_headers_handler.c index bc4c175..82b8f10 100644 --- a/src/response_headers_handler.c +++ b/src/response_headers_handler.c @@ -2,12 +2,14 @@ * response_headers_handler.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/s3.c b/src/s3.c index daa41a4..d1c0d03 100644 --- a/src/s3.c +++ b/src/s3.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ /** diff --git a/src/service.c b/src/service.c index 7c3d785..b6a1c10 100644 --- a/src/service.c +++ b/src/service.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/service_access_logging.c b/src/service_access_logging.c index 141db91..ce7ae6b 100644 --- a/src/service_access_logging.c +++ b/src/service_access_logging.c @@ -7,7 +7,9 @@ * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/simplexml.c b/src/simplexml.c index 658ccce..4f26101 100644 --- a/src/simplexml.c +++ b/src/simplexml.c @@ -2,12 +2,14 @@ * simplexml.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/testsimplexml.c b/src/testsimplexml.c index 6fb7dee..941302f 100644 --- a/src/testsimplexml.c +++ b/src/testsimplexml.c @@ -2,12 +2,14 @@ * testsimplexml.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include diff --git a/src/util.c b/src/util.c index 9c35934..7bdcf08 100644 --- a/src/util.c +++ b/src/util.c @@ -2,12 +2,14 @@ * util.c * * Copyright 2008 Bryan Ischo - * + * * This file is part of libs3. - * + * * libs3 is free software: you can redistribute it and/or modify it under the * terms of the GNU Lesser General Public License as published by the Free - * Software Foundation, version 3 of the License. + * Software Foundation, version 3 or above of the License. You can also + * redistribute and/or modify it under the terms of the GNU General Public + * License, version 2 or above of the License. * * In addition, as a special exception, the copyright holders give * permission to link the code of this library and its programs with the @@ -22,6 +24,10 @@ * version 3 along with libs3, in a file named COPYING. If not, see * . * + * You should also have received a copy of the GNU General Public License + * version 2 along with libs3, in a file named COPYING-GPLv2. If not, see + * . + * ************************************************************************** **/ #include From 7f2fb1be2aaef984cfa535b77eee3b3f14374ae2 Mon Sep 17 00:00:00 2001 From: Jan Engelhardt Date: Fri, 16 Nov 2018 12:50:22 +0100 Subject: [PATCH 54/57] build: support curl >= 7.62 libs3 fails to build with 7.62 currently: src/request.c: In function 'request_curl_code_to_status': src/request.c:1703:5: error: duplicate case value case CURLE_SSL_CACERT: src/request.c:1699:5: note: previously used here case CURLE_PEER_FAILED_VERIFICATION: --- src/request.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/request.c b/src/request.c index 4d43cf9..6d0a6c3 100644 --- a/src/request.c +++ b/src/request.c @@ -1719,7 +1719,9 @@ S3Status request_curl_code_to_status(CURLcode code) #else case CURLE_SSL_PEER_CERTIFICATE: #endif +#if LIBCURL_VERSION_NUM < 0x073e00 case CURLE_SSL_CACERT: +#endif return S3StatusServerFailedVerification; default: return S3StatusInternalError; From 4f2db1ebfa82cc4b364a891e1deeaa73dfda9c24 Mon Sep 17 00:00:00 2001 From: bji Date: Thu, 29 Nov 2018 11:45:06 -0800 Subject: [PATCH 55/57] Fix a bunch of code that I accepted via pull requests previously that were not buffer overflow safe. --- GNUmakefile | 3 +- src/request.c | 86 ++++++++++++++++++++++++++++++--------------------- 2 files changed, 52 insertions(+), 37 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 8ab4109..8bc19fb 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -142,8 +142,7 @@ ifndef CFLAGS endif endif -CFLAGS += -Wall -Werror -Wshadow -Wextra \ - -Iinc \ +CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ diff --git a/src/request.c b/src/request.c index 4d43cf9..d68d09e 100644 --- a/src/request.c +++ b/src/request.c @@ -132,7 +132,7 @@ typedef struct RequestComputedValues char rangeHeader[128]; // Authorization header - char authorizationHeader[1024]; + char authorizationHeader[4096]; // Request date stamp char requestDateISO8601[64]; @@ -292,8 +292,9 @@ static S3Status append_amz_header(RequestComputedValues *values, values->amzHeaders[values->amzHeadersCount++] = &(values->amzHeadersRaw[rawPos]); const char *headerStr = headerName; + + char headerNameWithPrefix[S3_MAX_METADATA_SIZE - sizeof(": v")]; if (addPrefix) { - char headerNameWithPrefix[S3_MAX_METADATA_SIZE - sizeof(": v")]; snprintf(headerNameWithPrefix, sizeof(headerNameWithPrefix), S3_METADATA_HEADER_NAME_PREFIX "%s", headerName); headerStr = headerNameWithPrefix; @@ -790,13 +791,13 @@ static void canonicalize_signature_headers(RequestComputedValues *values) // Canonicalizes the resource into params->canonicalizedResource static void canonicalize_resource(const S3BucketContext *context, const char *urlEncodedKey, - char *buffer) + char *buffer, unsigned int buffer_max) { int len = 0; *buffer = 0; -#define append(str) len += sprintf(&(buffer[len]), "%s", str) +#define append(str) len += snprintf(&(buffer[len]), buffer_max - len, "%s", str) if (context->uriStyle == S3UriStylePath) { if (context->bucketName && context->bucketName[0]) { @@ -814,8 +815,8 @@ static void canonicalize_resource(const S3BucketContext *context, #undef append } - -static void sort_query_string(const char *queryString, char *result) +static void sort_query_string(const char *queryString, char *result, + unsigned int result_size) { #ifdef SIGNATURE_DEBUG printf("\n--\nsort_and_urlencode\nqueryString: %s\n", queryString); @@ -830,10 +831,10 @@ static void sort_query_string(const char *queryString, char *result) const char* params[numParams]; - char tokenized[strlen(queryString) + 1]; - strncpy(tokenized, queryString, strlen(queryString) + 1); - - char *tok = tokenized; + // Where did strdup go?!?? + int queryStringLen = strlen(queryString); + char *tok = (char *) malloc(queryStringLen + 1); + strcpy(tok, queryString); const char *token = NULL; char *save = NULL; unsigned int i = 0; @@ -851,30 +852,39 @@ static void sort_query_string(const char *queryString, char *result) } #endif + // All params are urlEncoded +#define append(str) len += snprintf(&(result[len]), result_size - len, "%s", str) unsigned int pi = 0; + unsigned int len = 0; for (; pi < numParams; pi++) { - // All params are urlEncoded - strncat(result, params[pi], strlen(params[pi])); - strncat(result, "&", 1); + append(params[pi]); + append("&"); + } + // Take off the extra '&' + if (len > 0) { + result[len - 1] = 0; } - result[strlen(result) - 1] = '\0'; +#undef append + + free(tok); } // Canonicalize the query string part of the request into a buffer static void canonicalize_query_string(const char *queryParams, - const char *subResource, char *buffer) + const char *subResource, + char *buffer, unsigned int buffer_size) { int len = 0; *buffer = 0; -#define append(str) len += sprintf(&(buffer[len]), "%s", str) +#define append(str) len += snprintf(&(buffer[len]), buffer_size - len, "%s", str) if (queryParams && queryParams[0]) { char sorted[strlen(queryParams) * 2]; sorted[0] = '\0'; - sort_query_string(queryParams, sorted); + sort_query_string(queryParams, sorted, sizeof(sorted)); append(sorted); } @@ -990,12 +1000,13 @@ static S3Status compose_auth_header(const RequestParams *params, if (params->bucketContext.authRegion) { awsRegion = params->bucketContext.authRegion; } - char scope[SIGNATURE_SCOPE_SIZE + 1]; + char scope[sizeof(values->requestDateISO8601) + sizeof(awsRegion) + + sizeof("//s3/aws4_request") + 1]; snprintf(scope, sizeof(scope), "%.8s/%s/s3/aws4_request", values->requestDateISO8601, awsRegion); - char stringToSign[17 + 17 + SIGNATURE_SCOPE_SIZE + 1 - + strlen(canonicalRequestHashHex)]; + char stringToSign[17 + 17 + sizeof(values->requestDateISO8601) + + sizeof(scope) + sizeof(canonicalRequestHashHex) + 1]; snprintf(stringToSign, sizeof(stringToSign), "AWS4-HMAC-SHA256\n%s\n%s\n%s", values->requestDateISO8601, scope, canonicalRequestHashHex); @@ -1059,12 +1070,11 @@ static S3Status compose_auth_header(const RequestParams *params, "%s/%.8s/%s/s3/aws4_request", params->bucketContext.accessKeyId, values->requestDateISO8601, awsRegion); - snprintf( - values->authorizationHeader, - sizeof(values->authorizationHeader), - "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", - values->authCredential, values->signedHeaders, - values->requestSignatureHex); + snprintf(values->authorizationHeader, + sizeof(values->authorizationHeader), + "Authorization: AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + values->authCredential, values->signedHeaders, + values->requestSignatureHex); #ifdef SIGNATURE_DEBUG printf("--\nAuthorization Header:\n%s\n", values->authorizationHeader); @@ -1458,8 +1468,8 @@ S3Status request_api_initialize(const char *userAgentInfo, int flags, userAgentInfo = "Unknown"; } - char platform[96]; struct utsname utsn; + char platform[sizeof(utsn.sysname) + 1 + sizeof(utsn.machine) + 1]; if (uname(&utsn)) { snprintf(platform, sizeof(platform), "Unknown"); } @@ -1528,9 +1538,11 @@ static S3Status setup_request(const RequestParams *params, // Compute the canonicalized resource canonicalize_resource(¶ms->bucketContext, computed->urlEncodedKey, - computed->canonicalURI); + computed->canonicalURI, + sizeof(computed->canonicalURI)); canonicalize_query_string(params->queryParams, params->subResource, - computed->canonicalQueryString); + computed->canonicalQueryString, + sizeof(computed->canonicalQueryString)); // Compose Authorization header if ((status = compose_auth_header(params, computed)) != S3StatusOK) { @@ -1754,12 +1766,16 @@ S3Status S3_generate_authenticated_query_string } // Finally, compose the URI, with params - char queryParams[sizeof("X-Amz-Algorithm=AWS4-HMAC-SHA256") - + sizeof("&X-Amz-Credential=") + MAX_CREDENTIAL_SIZE - + sizeof("&X-Amz-Date=") + 16 + sizeof("&X-Amz-Expires=") + 6 - + sizeof("&X-Amz-SignedHeaders=") + 128 + sizeof("&X-Amz-Signature=") - + sizeof(computed.requestSignatureHex) + 1]; - + char queryParams[sizeof("X-Amz-Algorithm=AWS4-HMAC-SHA256") + + sizeof("&X-Amz-Credential=") + + sizeof(computed.authCredential) + + sizeof("&X-Amz-Date=") + + sizeof(computed.requestDateISO8601) + + sizeof("&X-Amz-Expires=") + 64 + + sizeof("&X-Amz-SignedHeaders=") + + sizeof(computed.signedHeaders) + + sizeof("&X-Amz-Signature=") + + sizeof(computed.requestSignatureHex) + 1]; snprintf(queryParams, sizeof(queryParams), "X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=%s" "&X-Amz-Date=%s&X-Amz-Expires=%d" From f7c1c488b0649b8fde556c76d4d46b5fddee4e49 Mon Sep 17 00:00:00 2001 From: Martin Prikryl Date: Mon, 3 Dec 2018 13:55:02 +0100 Subject: [PATCH 56/57] Memory leak tok is set NULL in strtok_r loop, so free was called for NULL, instead of the original pointer --- src/request.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/request.c b/src/request.c index 34f1f23..dd66863 100644 --- a/src/request.c +++ b/src/request.c @@ -833,7 +833,8 @@ static void sort_query_string(const char *queryString, char *result, // Where did strdup go?!?? int queryStringLen = strlen(queryString); - char *tok = (char *) malloc(queryStringLen + 1); + char *buf = (char *) malloc(queryStringLen + 1); + char *tok = buf; strcpy(tok, queryString); const char *token = NULL; char *save = NULL; @@ -866,7 +867,7 @@ static void sort_query_string(const char *queryString, char *result, } #undef append - free(tok); + free(buf); } From 98f667b248a7288c1941582897343171cfdf441c Mon Sep 17 00:00:00 2001 From: Bryan Ischo Date: Wed, 24 Apr 2024 14:24:21 -0700 Subject: [PATCH 57/57] Also build static version of binary. --- GNUmakefile | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 8bc19fb..226136d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -142,7 +142,7 @@ ifndef CFLAGS endif endif -CFLAGS += -Wall -Werror -Wshadow -Wextra -Iinc \ +CFLAGS += -Wall -Wshadow -Wextra -Iinc \ $(CURL_CFLAGS) $(LIBXML2_CFLAGS) \ -DLIBS3_VER_MAJOR=\"$(LIBS3_VER_MAJOR)\" \ -DLIBS3_VER_MINOR=\"$(LIBS3_VER_MINOR)\" \ @@ -168,7 +168,7 @@ all: exported test # Exported targets are the library and driver program .PHONY: exported -exported: libs3 s3 headers +exported: libs3 s3 s3-static headers # -------------------------------------------------------------------------- @@ -264,12 +264,17 @@ $(LIBS3_STATIC): $(LIBS3_SOURCES:%.c=$(BUILD)/obj/%.o) .PHONY: s3 s3: $(BUILD)/bin/s3 +s3-static: $(BUILD)/bin/s3-static $(BUILD)/bin/s3: $(BUILD)/obj/s3.o $(LIBS3_SHARED) $(QUIET_ECHO) $@: Building executable @ mkdir -p $(dir $@) $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) +$(BUILD)/bin/s3-static: $(BUILD)/obj/s3.o $(LIBS3_STATIC) + $(QUIET_ECHO) $@: Building static executable + @ mkdir -p $(dir $@) + $(VERBOSE_SHOW) $(CC) -o $@ $^ $(LDFLAGS) # -------------------------------------------------------------------------- # libs3 header targets