From 7b7291a00d26e15d12d8dcd3586622156b75e922 Mon Sep 17 00:00:00 2001 From: DenverM80 Date: Tue, 11 Jul 2017 13:02:35 -0600 Subject: [PATCH 1/3] Add test for max offset to not overflow buffer; fix put_bulk test regression --- src/CMakeLists.txt | 4 +++- src/ds3_init_requests.c | 13 ++++--------- test/bucket_tests.cpp | 20 +++++++++++++++++--- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 089e3d8b..7df52ba7 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -9,8 +9,10 @@ add_library(ds3 SHARED ds3_connection.h ds3_connection.c) if (WIN32) - #set(WINDOWS_VS_OUTPUT_DIR ${PROJECT_SOURCE_DIR}/win32/src/Debug) + set(CMAKE_BUILD_TYPE Release) set(WINDOWS_VS_OUTPUT_DIR ${PROJECT_SOURCE_DIR}/win32/src/Release) + #set(CMAKE_BUILD_TYPE Debug) + #set(WINDOWS_VS_OUTPUT_DIR ${PROJECT_SOURCE_DIR}/win32/src/Debug) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${WINDOWS_VS_OUTPUT_DIR}) set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${WINDOWS_VS_OUTPUT_DIR}) diff --git a/src/ds3_init_requests.c b/src/ds3_init_requests.c index 5732c32d..1f349439 100644 --- a/src/ds3_init_requests.c +++ b/src/ds3_init_requests.c @@ -26,12 +26,7 @@ #include "ds3_request.h" #include "ds3_net.h" -//The max size of an uint32_t should be 10 characters + NULL -static const char UNSIGNED_LONG_BASE_10[] = "4294967296"; -static const unsigned int UNSIGNED_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_BASE_10) + 1; -//The max size of an uint64_t should be 20 characters + NULL -static const char UNSIGNED_LONG_LONG_BASE_10[] = "18446744073709551615"; -static const unsigned int UNSIGNED_LONG_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_LONG_BASE_10) + 1; +static const unsigned int STRING_BUFFER_SIZE = 32; static char* _get_ds3_bucket_acl_permission_str(ds3_bucket_acl_permission input) { if (input == DS3_BUCKET_ACL_PERMISSION_LIST) { @@ -857,21 +852,21 @@ static void _set_query_param_flag(const ds3_request* _request, const char* key, } static void _set_query_param_uint64_t(const ds3_request* _request, const char* key, uint64_t value) { - char string_buffer[UNSIGNED_LONG_LONG_BASE_10_STR_LEN]; + char string_buffer[STRING_BUFFER_SIZE]; memset(string_buffer, 0, sizeof(string_buffer)); snprintf(string_buffer, sizeof(string_buffer), "%" PRIu64, value); _set_query_param(_request, key, string_buffer); } static void _set_query_param_int(const ds3_request* _request, const char* key, int value) { - char string_buffer[UNSIGNED_LONG_BASE_10_STR_LEN]; + char string_buffer[STRING_BUFFER_SIZE]; memset(string_buffer, 0, sizeof(string_buffer)); snprintf(string_buffer, sizeof(string_buffer), "%d", value); _set_query_param(_request, key, string_buffer); } static void _set_query_param_float(const ds3_request* _request, const char* key, float value) { - char string_buffer[UNSIGNED_LONG_BASE_10_STR_LEN]; + char string_buffer[STRING_BUFFER_SIZE]; memset(string_buffer, 0, sizeof(string_buffer)); snprintf(string_buffer, sizeof(string_buffer), "%f", value); _set_query_param(_request, key, string_buffer); diff --git a/test/bucket_tests.cpp b/test/bucket_tests.cpp index be2703a5..9c370e19 100644 --- a/test/bucket_tests.cpp +++ b/test/bucket_tests.cpp @@ -15,9 +15,23 @@ #include #include "ds3.h" +#include "ds3_request.h" #include "test.h" #include #include +#include + +BOOST_AUTO_TEST_CASE( init_put_object_offset_unsigned_long_long ) { + printf("-----Testing init PUT_OBJECT with max unsigned long long offset-------\n"); + + struct _ds3_request* request = ds3_init_put_object_request("Test_bucket", "test_object", 1024 * 1024); + ds3_request_set_offset(request, ULLONG_MAX); + + char* value = (char*)g_hash_table_lookup(request->query_params, "offset"); + printf(" request->query_params[\"offset\"]:%s\n", value); + BOOST_CHECK(value != NULL); + BOOST_CHECK(g_strcmp0(value, "18446744073709551615") == 0); +} BOOST_AUTO_TEST_CASE( bulk_put ) { printf("-----Testing Bulk PUT-------\n"); @@ -63,15 +77,15 @@ BOOST_AUTO_TEST_CASE( empty_folder ) { const char* objects[1] = {"empty-folder/"}; ds3_bulk_object_list_response* object_list = ds3_convert_object_list_from_strings(objects, 1); ds3_request* request = ds3_init_put_bulk_job_spectra_s3_request(bucket_name, object_list); - ds3_master_object_list_response* bulk_response = NULL; + ds3_master_object_list_response* bulk_response; error = ds3_put_bulk_job_spectra_s3_request(client, request, &bulk_response); handle_error(error); - ds3_master_object_list_response_free(bulk_response); request = ds3_init_put_object_request(bucket_name, objects[0], 0); ds3_request_set_job(request, bulk_response->job_id->value); - error = ds3_put_object_request(client, request, NULL, NULL); + error = ds3_put_object_request(client, request, NULL, NULL); ds3_request_free(request); + ds3_master_object_list_response_free(bulk_response); handle_error(error); ds3_list_bucket_result_response* response; From 939308be7483d4aef771ddd01b9d2ef3e77286b7 Mon Sep 17 00:00:00 2001 From: DenverM80 Date: Tue, 11 Jul 2017 13:55:50 -0600 Subject: [PATCH 2/3] Visual studio does not consider static const to be Constant; use #define instead --- src/ds3_init_requests.c | 2 +- src/ds3_requests.c | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/ds3_init_requests.c b/src/ds3_init_requests.c index 1f349439..d7f11f9e 100644 --- a/src/ds3_init_requests.c +++ b/src/ds3_init_requests.c @@ -26,7 +26,7 @@ #include "ds3_request.h" #include "ds3_net.h" -static const unsigned int STRING_BUFFER_SIZE = 32; +#define STRING_BUFFER_SIZE 32 static char* _get_ds3_bucket_acl_permission_str(ds3_bucket_acl_permission input) { if (input == DS3_BUCKET_ACL_PERMISSION_LIST) { diff --git a/src/ds3_requests.c b/src/ds3_requests.c index 81435f3b..780bb99d 100644 --- a/src/ds3_requests.c +++ b/src/ds3_requests.c @@ -34,13 +34,9 @@ #endif -//The max size of an uint32_t should be 10 characters + NULL -static const char UNSIGNED_LONG_BASE_10[] = "4294967296"; -static const unsigned int UNSIGNED_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_BASE_10) + 1; -//The max size of an uint64_t should be 20 characters + NULL -static const char UNSIGNED_LONG_LONG_BASE_10[] = "18446744073709551615"; -static const unsigned int UNSIGNED_LONG_LONG_BASE_10_STR_LEN = sizeof(UNSIGNED_LONG_LONG_BASE_10) + 1; - +//The max size of an uint32_t is 10 characters + NULL +//The max size of an uint64_t is 20 characters + NULL +#define STRING_BUFFER_SIZE 32 struct _ds3_metadata { @@ -408,7 +404,7 @@ static ds3_error* _get_request_xml_nodes( } static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_response* obj_list, object_list_type list_type, ds3_job_chunk_client_processing_order_guarantee order) { - char size_buff[UNSIGNED_LONG_LONG_BASE_10_STR_LEN]; + char size_buff[STRING_BUFFER_SIZE]; xmlDocPtr doc; ds3_bulk_object_response* obj; xmlNodePtr objects_node, object_node; @@ -431,7 +427,7 @@ static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_resp for (obj_index = 0; obj_index < obj_list->num_objects; obj_index++) { obj = obj_list->objects[obj_index]; - g_snprintf(size_buff, sizeof(char) * UNSIGNED_LONG_LONG_BASE_10_STR_LEN, "%" PRIu64, obj->length); + g_snprintf(size_buff, STRING_BUFFER_SIZE, "%" PRIu64, obj->length); object_node = xmlNewNode(NULL, (xmlChar*) "Object"); xmlAddChild(objects_node, object_node); @@ -448,7 +444,7 @@ static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_resp } static xmlDocPtr _generate_xml_complete_mpu(const ds3_complete_multipart_upload_response* mpu_list) { - char size_buff[UNSIGNED_LONG_LONG_BASE_10_STR_LEN]; + char size_buff[STRING_BUFFER_SIZE]; xmlDocPtr doc; ds3_multipart_upload_part_response* part; xmlNodePtr parts_node, part_node; @@ -464,7 +460,7 @@ static xmlDocPtr _generate_xml_complete_mpu(const ds3_complete_multipart_upload_ part_node = xmlNewNode(NULL, (xmlChar*) "Part"); xmlAddChild(parts_node, part_node); - g_snprintf(size_buff, sizeof(char) * UNSIGNED_LONG_LONG_BASE_10_STR_LEN, "%d", part->part_number); + g_snprintf(size_buff, STRING_BUFFER_SIZE, "%d", part->part_number); xmlNewTextChild(part_node, NULL, (xmlChar*) "PartNumber", (xmlChar*) size_buff); xmlNewTextChild(part_node, NULL, (xmlChar*) "ETag", (xmlChar*) part->etag->value); From 13c7823484517fc1e9a441b02c7d55f37f7e6540 Mon Sep 17 00:00:00 2001 From: DenverM80 Date: Tue, 11 Jul 2017 15:47:56 -0600 Subject: [PATCH 3/3] Fix printing of unsigned long long in a platform independant manner --- src/ds3_init_requests.c | 15 ++++++++++++--- src/ds3_requests.c | 8 ++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/src/ds3_init_requests.c b/src/ds3_init_requests.c index d7f11f9e..eeb9bd89 100644 --- a/src/ds3_init_requests.c +++ b/src/ds3_init_requests.c @@ -26,6 +26,15 @@ #include "ds3_request.h" #include "ds3_net.h" +#ifdef _WIN32 + #include + #ifndef PRIu64 + #define PRIu64 "I64u" + #endif +#else + #include +#endif + #define STRING_BUFFER_SIZE 32 static char* _get_ds3_bucket_acl_permission_str(ds3_bucket_acl_permission input) { @@ -854,21 +863,21 @@ static void _set_query_param_flag(const ds3_request* _request, const char* key, static void _set_query_param_uint64_t(const ds3_request* _request, const char* key, uint64_t value) { char string_buffer[STRING_BUFFER_SIZE]; memset(string_buffer, 0, sizeof(string_buffer)); - snprintf(string_buffer, sizeof(string_buffer), "%" PRIu64, value); + g_snprintf(string_buffer, sizeof(string_buffer), "%" PRIu64, value); _set_query_param(_request, key, string_buffer); } static void _set_query_param_int(const ds3_request* _request, const char* key, int value) { char string_buffer[STRING_BUFFER_SIZE]; memset(string_buffer, 0, sizeof(string_buffer)); - snprintf(string_buffer, sizeof(string_buffer), "%d", value); + g_snprintf(string_buffer, sizeof(string_buffer), "%d", value); _set_query_param(_request, key, string_buffer); } static void _set_query_param_float(const ds3_request* _request, const char* key, float value) { char string_buffer[STRING_BUFFER_SIZE]; memset(string_buffer, 0, sizeof(string_buffer)); - snprintf(string_buffer, sizeof(string_buffer), "%f", value); + g_snprintf(string_buffer, sizeof(string_buffer), "%f", value); _set_query_param(_request, key, string_buffer); } diff --git a/src/ds3_requests.c b/src/ds3_requests.c index 780bb99d..6b4744ce 100644 --- a/src/ds3_requests.c +++ b/src/ds3_requests.c @@ -28,9 +28,12 @@ #include "ds3_utils.h" #ifdef _WIN32 -#include + #include + #ifndef PRIu64 + #define PRIu64 "I64u" + #endif #else -#include + #include #endif @@ -427,6 +430,7 @@ static xmlDocPtr _generate_xml_bulk_objects_list(const ds3_bulk_object_list_resp for (obj_index = 0; obj_index < obj_list->num_objects; obj_index++) { obj = obj_list->objects[obj_index]; + memset(size_buff, 0, sizeof(size_buff)); g_snprintf(size_buff, STRING_BUFFER_SIZE, "%" PRIu64, obj->length); object_node = xmlNewNode(NULL, (xmlChar*) "Object");