vasnprintf: Fix heap memory overrun bug.

Reported by Ben Pfaff <blp@cs.stanford.edu> in
<https://lists.gnu.org/archive/html/bug-gnulib/2018-09/msg00107.html>.

* lib/vasnprintf.c (convert_to_decimal): Allocate one more byte of
memory.
* tests/test-vasnprintf.c (test_function): Add another test.

ChangeLog

@@ -1,3 +1,12 @@
+2018-09-23  Bruno Haible  <bruno@clisp.org>
+
+	vasnprintf: Fix heap memory overrun bug.
+	Reported by Ben Pfaff <blp@cs.stanford.edu> in
+	<https://lists.gnu.org/archive/html/bug-gnulib/2018-09/msg00107.html>.
+	* lib/vasnprintf.c (convert_to_decimal): Allocate one more byte of
+	memory.
+	* tests/test-vasnprintf.c (test_function): Add another test.
+
 2018-09-19  Paul Eggert  <eggert@cs.ucla.edu>
 
 	maint: mktime.c now shared with glibc

lib/vasnprintf.c

@@ -860,7 +860,9 @@ convert_to_decimal (mpn_t a, size_t extra_zeroes)
   size_t a_len = a.nlimbs;
   /* 0.03345 is slightly larger than log(2)/(9*log(10)).  */
   size_t c_len = 9 * ((size_t)(a_len * (GMP_LIMB_BITS * 0.03345f)) + 1);
-  char *c_ptr = (char *) malloc (xsum (c_len, extra_zeroes));
+  /* We need extra_zeroes bytes for zeroes, followed by c_len bytes for the
+     digits of a, followed by 1 byte for the terminating NUL.  */
+  char *c_ptr = (char *) malloc (xsum (xsum (extra_zeroes, c_len), 1));
   if (c_ptr != NULL)
     {
       char *d_ptr = c_ptr;

tests/test-vasnprintf.c

@@ -53,7 +53,26 @@ test_function (char * (*my_asnprintf) (char *, size_t *, const char *, ...))
       ASSERT (result != NULL);
       ASSERT (strcmp (result, "12345") == 0);
       ASSERT (length == 5);
-      if (size < 6)
+      if (size < 5 + 1)
+        ASSERT (result != buf);
+      ASSERT (memcmp (buf + size, &"DEADBEEF"[size], 8 - size) == 0);
+      if (result != buf)
+        free (result);
+    }
+
+  /* Note: This test assumes IEEE 754 representation of 'double' floats.  */
+  for (size = 0; size <= 8; size++)
+    {
+      size_t length;
+      char *result;
+
+      memcpy (buf, "DEADBEEF", 8);
+      length = size;
+      result = my_asnprintf (buf, &length, "%2.0f", 1.6314159265358979e+125);
+      ASSERT (result != NULL);
+      ASSERT (strcmp (result, "163141592653589790215729350939528493057529598899734151772468186268423257777068536614838678161083520756952076273094236944990208") == 0);
+      ASSERT (length == 126);
+      if (size < 126 + 1)
         ASSERT (result != buf);
       ASSERT (memcmp (buf + size, &"DEADBEEF"[size], 8 - size) == 0);
       if (result != buf)