[klibc] tests: Add test for malloc size arithmetic

It has been reported that klibc's malloc() and calloc() are
vulnerable to integer overflows.  Add test cases demonstrating
some of these.

Signed-off-by: Ben Hutchings <ben@decadent.org.uk>

usr/klibc/tests/Kbuild

@@ -10,6 +10,11 @@ test-files := $(notdir $(test-files))
 # of useless warnings unless we tell it not to.
 KLIBCCFLAGS_testvsnp.o := -Wno-format
 
+# This deliberately calls malloc() with unreasonably large values.  We
+# can't use cc-disable-warning here as the option to *enable* this
+# warning requires a value.
+KLIBCCFLAGS_malloctest3.o := $(call cc-option,-Wno-alloc-size-larger-than)
+
 static-y := $(test-files:.c=)
 shared-y := $(addsuffix .shared, $(static-y))
 
@@ -24,6 +29,7 @@ idtest.shared-y		:= idtest.o
 lseek.shared-y		:= lseek.o
 malloctest.shared-y	:= malloctest.o
 malloctest2.shared-y	:= malloctest2.o
+malloctest3.shared-y	:= malloctest3.o
 memstrtest.shared-y	:= memstrtest.o
 microhello.shared-y	:= microhello.o
 minihello.shared-y	:= minihello.o

usr/klibc/tests/malloctest3.c

@@ -0,0 +1,57 @@
+#include <assert.h>
+#include <errno.h>
+#include <limits.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int main(void)
+{
+	void *p;
+
+	/* Our implementation should always return NULL */
+	errno = 0;
+	p = malloc(0);
+	assert(p == NULL);
+	assert(errno == 0);
+
+	/* These sizes won't fit in memory, so should always fail */
+	errno = 0;
+	p = malloc(SIZE_MAX);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+	errno = 0;
+	p = malloc(SIZE_MAX - 0x10000);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+
+#if SIZE_MAX > 0x100000000
+	/* We should be able to allocate 4 GB + 1 */
+	p = malloc(0x100000001);
+	assert(p != NULL);
+	((volatile char *)p)[0x100000000] = 1;
+	free(p);
+
+	/* calloc() should detect multiplication overflow */
+	errno = 0;
+	p = calloc(0x100000000, 0x100000000);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+	errno = 0;
+	p = calloc(0x100000001, 0x100000001);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+#else
+	/* calloc() should detect multiplication overflow */
+	errno = 0;
+	p = calloc(0x10000, 0x10000);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+	errno = 0;
+	p = calloc(0x10001, 0x10001);
+	assert(p == NULL);
+	assert(errno == ENOMEM);
+#endif
+
+	return 0;
+}