Follow ups for 64 bit change

c/tests/test_core.c

@@ -1,7 +1,7 @@
 /*
  * MIT License
  *
- * Copyright (c) 2019 Tskit Developers
+ * Copyright (c) 2019-2021 Tskit Developers
  *
  * Permission is hereby granted, free of charge, to any person obtaining a copy
  * of this software and associated documentation files (the "Software"), to deal
@@ -358,6 +358,19 @@ test_malloc_zero(void)
     free(p);
 }
 
+static void
+test_malloc_overflow(void)
+{
+#if TSK_MAX_SIZE > SIZE_MAX
+    tsk_size_t size_max = SIZE_MAX;
+    void *p = tsk_malloc(size_max + 1);
+    CU_ASSERT_FATAL(p == NULL);
+
+    p = tsk_calloc(size_max + 1, 1);
+    CU_ASSERT_FATAL(p == NULL);
+#endif
+}
+
 int
 main(int argc, char **argv)
 {
@@ -369,6 +382,7 @@ main(int argc, char **argv)
         { "test_blkalloc", test_blkalloc },
         { "test_unknown_time", test_unknown_time },
         { "test_malloc_zero", test_malloc_zero },
+        { "test_malloc_overflow", test_malloc_overflow },
         { NULL, NULL },
     };
 

c/tskit/core.c

@@ -717,9 +717,11 @@ tsk_malloc(tsk_size_t size)
     if (size == 0) {
         size = 1;
     }
-    /* TODO
-     * 1. check if size > SIZE_MAX on 32 bit
-     */
+#if TSK_MAX_SIZE > SIZE_MAX
+    if (size > SIZE_MAX) {
+        return NULL;
+    }
+#endif
     return malloc((size_t) size);
 }
 
@@ -731,13 +733,23 @@ tsk_realloc(void *ptr, tsk_size_t size)
     return realloc(ptr, (size_t) size);
 }
 
+/* We keep the size argument here as a size_t because we'd have to
+ * cast the outputs of sizeof() otherwise, which would lead to
+ * less readable code. We need to be careful to use calloc within
+ * the library accordingly, so that size can't overflow on 32 bit.
+ */
 void *
 tsk_calloc(tsk_size_t n, size_t size)
 {
     /* Avoid calloc(0) as it's not portable */
     if (n == 0) {
         n = 1;
     }
+#if TSK_MAX_SIZE > SIZE_MAX
+    if (n > SIZE_MAX) {
+        return NULL;
+    }
+#endif
     return calloc((size_t) n, size);
 }