* malloc/malloc.c (sYSMALLOc): Avoid infinite loop if MMAP

	keeps failing and heap growth or new heap creation isn't
	successful either.
	* malloc/tst-malloc.c (main): Add new tests.

ChangeLog

@@ -1,3 +1,10 @@
+2006-08-24  Jakub Jelinek  <jakub@redhat.com>
+
+	* malloc/malloc.c (sYSMALLOc): Avoid infinite loop if MMAP
+	keeps failing and heap growth or new heap creation isn't
+	successful either.
+	* malloc/tst-malloc.c (main): Add new tests.
+
 2006-08-24  Ulrich Drepper  <drepper@redhat.com>
 
 	[BZ #2734]

malloc/malloc.c

@@ -2860,6 +2860,7 @@ static Void_t* sYSMALLOc(nb, av) INTERNAL_SIZE_T nb; mstate av;
   unsigned long   sum;            /* for updating stats */
 
   size_t          pagemask  = mp_.pagesize - 1;
+  bool            tried_mmap = false;
 
 
 #if HAVE_MMAP
@@ -2883,6 +2884,7 @@ static Void_t* sYSMALLOc(nb, av) INTERNAL_SIZE_T nb; mstate av;
       is no following chunk whose prev_size field could be used.
     */
     size = (nb + SIZE_SZ + MALLOC_ALIGN_MASK + pagemask) & ~pagemask;
+    tried_mmap = true;
 
     /* Don't try if size wraps around 0 */
     if ((unsigned long)(size) > (unsigned long)(nb)) {
@@ -3006,7 +3008,7 @@ static Void_t* sYSMALLOc(nb, av) INTERNAL_SIZE_T nb; mstate av;
 	set_foot(old_top, (old_size + 2*SIZE_SZ));
       }
     }
-    else
+    else if (!tried_mmap)
       /* We can at least try to use to mmap memory.  */
       goto try_mmap;
 

malloc/tst-malloc.c

@@ -33,7 +33,7 @@ merror (const char *msg)
 int
 main (void)
 {
-  void *p;
+  void *p, *q;
   int save;
 
   errno = 0;
@@ -64,5 +64,15 @@ main (void)
   if (p != NULL)
     merror ("realloc (p, 0) failed.");
 
+  p = malloc (513 * 1024);
+  if (p == NULL)
+    merror ("malloc (513K) failed.");
+
+  q = malloc (-512 * 1024);
+  if (q != NULL)
+    merror ("malloc (-512K) succeeded.");
+
+  free (p);
+
   return errors != 0;
 }