Merge pull request #186 from PJK/pk/aa32

Fixed handling of items that exceed the host size_t range

CHANGELOG.md

@@ -2,6 +2,10 @@ Next
 ---------------------
 - Improved pkg-config paths handling [[#164]](https://github.com/PJK/libcbor/pull/164) (by [jtojnar@](https://github.com/jtojnar))
 - Use explicit math.h linkage [[#170]](https://github.com/PJK/libcbor/pull/170)
+- BREAKING: Fixed handling of items that exceed the host size_t range [[#186]](https://github.com/PJK/libcbor/pull/186hg)  
+    - Callbacks for bytestrings, strings, arrays, and maps use uint64_t instead of size_t to allow handling of large items that exceed size_t even if size_t < uint64_t
+    - cbor_decode explicitly checks size to avoid overflows (previously broken, potentially resulting in erroneous decoding on affected systems)
+    - The change should be a noop for 64b systems 
 
 0.8.0 (2020-09-20)
 ---------------------

CMakeLists.txt

@@ -85,7 +85,7 @@ set(CMAKE_EXE_LINKER_FLAGS_DEBUG "-g")
 include(CheckTypeSize)
 check_type_size("size_t" SIZEOF_SIZE_T)
 if(SIZEOF_SIZE_T LESS 8)
-    message(WARNING "Your size_t is less than 8 bytes. Long items with 64b length specifiers might not work as expected. Make sure to run the tests!")
+    message(WARNING "Your size_t is less than 8 bytes. Decoding of huge items that would exceed the memory address space will always fail. Consider implementing a custom streaming decoder if you need to deal with huge items.")
 else()
     add_definitions(-DEIGHT_BYTE_SIZE_T)
 endif()

src/cbor/callbacks.c

@@ -33,23 +33,23 @@ void cbor_null_negint64_callback(void *_CBOR_UNUSED(_ctx),
 
 void cbor_null_string_callback(void *_CBOR_UNUSED(_ctx),
                                cbor_data _CBOR_UNUSED(_val),
-                               size_t _CBOR_UNUSED(_val2)) {}
+                               uint64_t _CBOR_UNUSED(_val2)) {}
 
 void cbor_null_string_start_callback(void *_CBOR_UNUSED(_ctx)) {}
 
 void cbor_null_byte_string_callback(void *_CBOR_UNUSED(_ctx),
                                     cbor_data _CBOR_UNUSED(_val),
-                                    size_t _CBOR_UNUSED(_val2)) {}
+                                    uint64_t _CBOR_UNUSED(_val2)) {}
 
 void cbor_null_byte_string_start_callback(void *_CBOR_UNUSED(_ctx)) {}
 
 void cbor_null_array_start_callback(void *_CBOR_UNUSED(_ctx),
-                                    size_t _CBOR_UNUSED(_val)) {}
+                                    uint64_t _CBOR_UNUSED(_val)) {}
 
 void cbor_null_indef_array_start_callback(void *_CBOR_UNUSED(_ctx)) {}
 
 void cbor_null_map_start_callback(void *_CBOR_UNUSED(_ctx),
-                                  size_t _CBOR_UNUSED(_val)) {}
+                                  uint64_t _CBOR_UNUSED(_val)) {}
 
 void cbor_null_indef_map_start_callback(void *_CBOR_UNUSED(_ctx)) {}
 

src/cbor/callbacks.h

@@ -8,6 +8,8 @@
 #ifndef LIBCBOR_CALLBACKS_H
 #define LIBCBOR_CALLBACKS_H
 
+#include <stdint.h>
+
 #include "cbor/cbor_export.h"
 #include "cbor/common.h"
 
@@ -31,10 +33,10 @@ typedef void (*cbor_int64_callback)(void *, uint64_t);
 typedef void (*cbor_simple_callback)(void *);
 
 /** Callback prototype */
-typedef void (*cbor_string_callback)(void *, cbor_data, size_t);
+typedef void (*cbor_string_callback)(void *, cbor_data, uint64_t);
 
 /** Callback prototype */
-typedef void (*cbor_collection_callback)(void *, size_t);
+typedef void (*cbor_collection_callback)(void *, uint64_t);
 
 /** Callback prototype */
 typedef void (*cbor_float_callback)(void *, float);
@@ -130,25 +132,25 @@ CBOR_EXPORT void cbor_null_negint32_callback(void *, uint32_t);
 CBOR_EXPORT void cbor_null_negint64_callback(void *, uint64_t);
 
 /** Dummy callback implementation - does nothing */
-CBOR_EXPORT void cbor_null_string_callback(void *, cbor_data, size_t);
+CBOR_EXPORT void cbor_null_string_callback(void *, cbor_data, uint64_t);
 
 /** Dummy callback implementation - does nothing */
 CBOR_EXPORT void cbor_null_string_start_callback(void *);
 
 /** Dummy callback implementation - does nothing */
-CBOR_EXPORT void cbor_null_byte_string_callback(void *, cbor_data, size_t);
+CBOR_EXPORT void cbor_null_byte_string_callback(void *, cbor_data, uint64_t);
 
 /** Dummy callback implementation - does nothing */
 CBOR_EXPORT void cbor_null_byte_string_start_callback(void *);
 
 /** Dummy callback implementation - does nothing */
-CBOR_EXPORT void cbor_null_array_start_callback(void *, size_t);
+CBOR_EXPORT void cbor_null_array_start_callback(void *, uint64_t);
 
 /** Dummy callback implementation - does nothing */
 CBOR_EXPORT void cbor_null_indef_array_start_callback(void *);
 
 /** Dummy callback implementation - does nothing */
-CBOR_EXPORT void cbor_null_map_start_callback(void *, size_t);
+CBOR_EXPORT void cbor_null_map_start_callback(void *, uint64_t);
 
 /** Dummy callback implementation - does nothing */
 CBOR_EXPORT void cbor_null_indef_map_start_callback(void *);

src/cbor/data.h

@@ -86,6 +86,11 @@ typedef enum {
   CBOR_CTRL_UNDEF = 23
 } _cbor_ctrl;
 
+// Metadata items use size_t (instead of uint64_t) because items in memory take
+// up at least 1B per entry or string byte, so if size_t is narrower than
+// uint64_t, we wouldn't be able to create them in the first place and can save
+// some space.
+
 /** Integers specific metadata */
 struct _cbor_int_metadata {
   cbor_int_width width;

src/cbor/internal/builder_callbacks.c

@@ -95,6 +95,16 @@ void _cbor_builder_append(cbor_item_t *item,
     }                              \
   } while (0)
 
+// Check that the length fits into size_t. If not, we cannot possibly allocate
+// the required memory and should fail fast.
+#define CHECK_LENGTH(ctx, length)  \
+  do {                             \
+    if (length > SIZE_MAX) {       \
+      ctx->creation_failed = true; \
+      return;                      \
+    }                              \
+  } while (0)
+
 #define PUSH_CTX_STACK(ctx, res, subitems)                     \
   do {                                                         \
     if (_cbor_stack_push(ctx->stack, res, subitems) == NULL) { \
@@ -176,8 +186,9 @@ void cbor_builder_negint64_callback(void *context, uint64_t value) {
 }
 
 void cbor_builder_byte_string_callback(void *context, cbor_data data,
-                                       size_t length) {
+                                       uint64_t length) {
   struct _cbor_decoder_context *ctx = context;
+  CHECK_LENGTH(ctx, length);
   unsigned char *new_handle = _CBOR_MALLOC(length);
   if (new_handle == NULL) {
     ctx->creation_failed = true;
@@ -215,17 +226,18 @@ void cbor_builder_byte_string_start_callback(void *context) {
 }
 
 void cbor_builder_string_callback(void *context, cbor_data data,
-                                  size_t length) {
+                                  uint64_t length) {
   struct _cbor_decoder_context *ctx = context;
+  CHECK_LENGTH(ctx, length);
   struct _cbor_unicode_status unicode_status;
-
-  size_t codepoint_count =
+  uint64_t codepoint_count =
       _cbor_unicode_codepoint_count(data, length, &unicode_status);
 
   if (unicode_status.status == _CBOR_UNICODE_BADCP) {
     ctx->syntax_error = true;
     return;
   }
+  assert(codepoint_count <= length);
 
   unsigned char *new_handle = _CBOR_MALLOC(length);
 
@@ -264,8 +276,9 @@ void cbor_builder_string_start_callback(void *context) {
   PUSH_CTX_STACK(ctx, res, 0);
 }
 
-void cbor_builder_array_start_callback(void *context, size_t size) {
+void cbor_builder_array_start_callback(void *context, uint64_t size) {
   struct _cbor_decoder_context *ctx = context;
+  CHECK_LENGTH(ctx, size);
   cbor_item_t *res = cbor_new_definite_array(size);
   CHECK_RES(ctx, res);
   if (size > 0) {
@@ -289,8 +302,9 @@ void cbor_builder_indef_map_start_callback(void *context) {
   PUSH_CTX_STACK(ctx, res, 0);
 }
 
-void cbor_builder_map_start_callback(void *context, size_t size) {
+void cbor_builder_map_start_callback(void *context, uint64_t size) {
   struct _cbor_decoder_context *ctx = context;
+  CHECK_LENGTH(ctx, size);
   cbor_item_t *res = cbor_new_definite_map(size);
   CHECK_RES(ctx, res);
   if (size > 0) {

src/cbor/internal/builder_callbacks.h

@@ -42,19 +42,19 @@ void cbor_builder_negint32_callback(void *, uint32_t);
 
 void cbor_builder_negint64_callback(void *, uint64_t);
 
-void cbor_builder_string_callback(void *, cbor_data, size_t);
+void cbor_builder_string_callback(void *, cbor_data, uint64_t);
 
 void cbor_builder_string_start_callback(void *);
 
-void cbor_builder_byte_string_callback(void *, cbor_data, size_t);
+void cbor_builder_byte_string_callback(void *, cbor_data, uint64_t);
 
 void cbor_builder_byte_string_start_callback(void *);
 
-void cbor_builder_array_start_callback(void *, size_t);
+void cbor_builder_array_start_callback(void *, uint64_t);
 
 void cbor_builder_indef_array_start_callback(void *);
 
-void cbor_builder_map_start_callback(void *, size_t);
+void cbor_builder_map_start_callback(void *, uint64_t);
 
 void cbor_builder_indef_map_start_callback(void *);
 

src/cbor/internal/unicode.c

@@ -65,12 +65,12 @@ uint32_t _cbor_unicode_decode(uint32_t* state, uint32_t* codep, uint32_t byte) {
   return *state;
 }
 
-size_t _cbor_unicode_codepoint_count(cbor_data source, size_t source_length,
-                                     struct _cbor_unicode_status* status) {
+uint64_t _cbor_unicode_codepoint_count(cbor_data source, uint64_t source_length,
+                                       struct _cbor_unicode_status* status) {
   *status =
       (struct _cbor_unicode_status){.location = 0, .status = _CBOR_UNICODE_OK};
   uint32_t codepoint, state = UTF8_ACCEPT, res;
-  size_t pos = 0, count = 0;
+  uint64_t pos = 0, count = 0;
 
   for (; pos < source_length; pos++) {
     res = _cbor_unicode_decode(&state, &codepoint, source[pos]);

src/cbor/internal/unicode.h

@@ -19,11 +19,11 @@ enum _cbor_unicode_status_error { _CBOR_UNICODE_OK, _CBOR_UNICODE_BADCP };
 /** Signals unicode validation error and possibly its location */
 struct _cbor_unicode_status {
   enum _cbor_unicode_status_error status;
-  size_t location;
+  uint64_t location;
 };
 
-size_t _cbor_unicode_codepoint_count(cbor_data source, size_t source_length,
-                                     struct _cbor_unicode_status* status);
+uint64_t _cbor_unicode_codepoint_count(cbor_data source, uint64_t source_length,
+                                       struct _cbor_unicode_status* status);
 
 #ifdef __cplusplus
 }