bmalloc: per-thread cache data structure should be smaller

https://bugs.webkit.org/show_bug.cgi?id=150218

Reviewed by Andreas Kling.

Reduce the number of entries in the range cache because it's really
big, and the bigness only helps in cases of serious fragmentation, and
it only saves us a little bit of lock acquisition time.

* bmalloc/Allocator.cpp:
(bmalloc::Allocator::scavenge):
(bmalloc::Allocator::refillAllocatorSlowCase):
(bmalloc::Allocator::refillAllocator):
(bmalloc::Allocator::allocateLarge):
(bmalloc::Allocator::allocateSlowCase):
(bmalloc::Allocator::allocateBumpRangeSlowCase): Deleted.
(bmalloc::Allocator::allocateBumpRange): Deleted.
* bmalloc/Allocator.h: Pass through the empty allocator and the range
cache when refilling, and refill both. Otherwise, we always immediately
pop the last item in the range cache, wasting that slot of capacity.

* bmalloc/Heap.cpp:
(bmalloc::Heap::allocateSmallBumpRanges):
(bmalloc::Heap::allocateMediumBumpRanges): Account for the fact that
the range cache is no longer big enough to guarantee that it can hold
all the ranges in a page.

(bmalloc::Heap::refillSmallBumpRangeCache): Deleted.
(bmalloc::Heap::refillMediumBumpRangeCache): Deleted.

* bmalloc/Heap.h: Move VMHeap to the end of the object because it
contains a lot of unused / wasted space, and we want to pack our data
together in memory.

* bmalloc/Sizes.h: Make the range cache smaller.


Canonical link: https://commits.webkit.org/168372@main
git-svn-id: https://svn.webkit.org/repository/webkit/trunk@191196 268f45cc-cd09-0410-ab3c-d52691b4dbfc

Source/bmalloc/ChangeLog

@@ -1,3 +1,41 @@
+2015-10-15  Geoffrey Garen  <ggaren@apple.com>
+
+        bmalloc: per-thread cache data structure should be smaller
+        https://bugs.webkit.org/show_bug.cgi?id=150218
+
+        Reviewed by Andreas Kling.
+
+        Reduce the number of entries in the range cache because it's really
+        big, and the bigness only helps in cases of serious fragmentation, and
+        it only saves us a little bit of lock acquisition time.
+
+        * bmalloc/Allocator.cpp:
+        (bmalloc::Allocator::scavenge):
+        (bmalloc::Allocator::refillAllocatorSlowCase):
+        (bmalloc::Allocator::refillAllocator):
+        (bmalloc::Allocator::allocateLarge):
+        (bmalloc::Allocator::allocateSlowCase):
+        (bmalloc::Allocator::allocateBumpRangeSlowCase): Deleted.
+        (bmalloc::Allocator::allocateBumpRange): Deleted.
+        * bmalloc/Allocator.h: Pass through the empty allocator and the range
+        cache when refilling, and refill both. Otherwise, we always immediately
+        pop the last item in the range cache, wasting that slot of capacity.
+
+        * bmalloc/Heap.cpp:
+        (bmalloc::Heap::allocateSmallBumpRanges):
+        (bmalloc::Heap::allocateMediumBumpRanges): Account for the fact that
+        the range cache is no longer big enough to guarantee that it can hold
+        all the ranges in a page.
+
+        (bmalloc::Heap::refillSmallBumpRangeCache): Deleted.
+        (bmalloc::Heap::refillMediumBumpRangeCache): Deleted.
+
+        * bmalloc/Heap.h: Move VMHeap to the end of the object because it
+        contains a lot of unused / wasted space, and we want to pack our data
+        together in memory.
+
+        * bmalloc/Sizes.h: Make the range cache smaller.
+
 2015-10-13  Chris Dumez  <cdumez@apple.com>
 
         Avoid useless copies in range-loops that are using 'auto'

Source/bmalloc/bmalloc/Allocator.cpp

@@ -203,25 +203,23 @@ void Allocator::scavenge()
     }
 }
 
-NO_INLINE BumpRange Allocator::allocateBumpRangeSlowCase(size_t sizeClass)
+NO_INLINE void Allocator::refillAllocatorSlowCase(BumpAllocator& allocator, size_t sizeClass)
 {
     BumpRangeCache& bumpRangeCache = m_bumpRangeCaches[sizeClass];
 
     std::lock_guard<StaticMutex> lock(PerProcess<Heap>::mutex());
     if (sizeClass <= bmalloc::sizeClass(smallMax))
-        PerProcess<Heap>::getFastCase()->refillSmallBumpRangeCache(lock, sizeClass, bumpRangeCache);
+        PerProcess<Heap>::getFastCase()->allocateSmallBumpRanges(lock, sizeClass, allocator, bumpRangeCache);
     else
-        PerProcess<Heap>::getFastCase()->refillMediumBumpRangeCache(lock, sizeClass, bumpRangeCache);
-
-    return bumpRangeCache.pop();
+        PerProcess<Heap>::getFastCase()->allocateMediumBumpRanges(lock, sizeClass, allocator, bumpRangeCache);
 }
 
-INLINE BumpRange Allocator::allocateBumpRange(size_t sizeClass)
+INLINE void Allocator::refillAllocator(BumpAllocator& allocator, size_t sizeClass)
 {
     BumpRangeCache& bumpRangeCache = m_bumpRangeCaches[sizeClass];
     if (!bumpRangeCache.size())
-        return allocateBumpRangeSlowCase(sizeClass);
-    return bumpRangeCache.pop();
+        return refillAllocatorSlowCase(allocator, sizeClass);
+    return allocator.refill(bumpRangeCache.pop());
 }
 
 NO_INLINE void* Allocator::allocateLarge(size_t size)
@@ -246,7 +244,7 @@ void* Allocator::allocateSlowCase(size_t size)
     if (size <= mediumMax) {
         size_t sizeClass = bmalloc::sizeClass(size);
         BumpAllocator& allocator = m_bumpAllocators[sizeClass];
-        allocator.refill(allocateBumpRange(sizeClass));
+        refillAllocator(allocator, sizeClass);
         return allocator.allocate();
     }
 

Source/bmalloc/bmalloc/Allocator.h

@@ -56,8 +56,8 @@ class Allocator {
     void* allocateLarge(size_t);
     void* allocateXLarge(size_t);
 
-    BumpRange allocateBumpRange(size_t sizeClass);
-    BumpRange allocateBumpRangeSlowCase(size_t sizeClass);
+    void refillAllocator(BumpAllocator&, size_t sizeClass);
+    void refillAllocatorSlowCase(BumpAllocator&, size_t sizeClass);
 
     std::array<BumpAllocator, mediumMax / alignment> m_bumpAllocators;
     std::array<BumpRangeCache, mediumMax / alignment> m_bumpRangeCaches;

Source/bmalloc/bmalloc/Heap.cpp

@@ -24,6 +24,7 @@
  */
 
 #include "Heap.h"
+#include "BumpAllocator.h"
 #include "LargeChunk.h"
 #include "LargeObject.h"
 #include "Line.h"
@@ -119,7 +120,7 @@ void Heap::scavengeLargeObjects(std::unique_lock<StaticMutex>& lock, std::chrono
     }
 }
 
-void Heap::refillSmallBumpRangeCache(std::lock_guard<StaticMutex>& lock, size_t sizeClass, BumpRangeCache& rangeCache)
+void Heap::allocateSmallBumpRanges(std::lock_guard<StaticMutex>& lock, size_t sizeClass, BumpAllocator& allocator, BumpRangeCache& rangeCache)
 {
     BASSERT(!rangeCache.size());
     SmallPage* page = allocateSmallPage(lock, sizeClass);
@@ -135,6 +136,12 @@ void Heap::refillSmallBumpRangeCache(std::lock_guard<StaticMutex>& lock, size_t
         if (lines[lineNumber].refCount(lock))
             continue;
 
+        // In a fragmented page, some free ranges might not fit in the cache.
+        if (rangeCache.size() == rangeCache.capacity()) {
+            m_smallPagesWithFreeLines[sizeClass].push(page);
+            return;
+        }
+
         LineMetadata& lineMetadata = m_smallLineMetadata[sizeClass][lineNumber];
         char* begin = lines[lineNumber].begin() + lineMetadata.startOffset;
         unsigned short objectCount = lineMetadata.objectCount;
@@ -152,11 +159,14 @@ void Heap::refillSmallBumpRangeCache(std::lock_guard<StaticMutex>& lock, size_t
             page->ref(lock);
         }
 
-        rangeCache.push({ begin, objectCount });
+        if (!allocator.canAllocate())
+            allocator.refill({ begin, objectCount });
+        else
+            rangeCache.push({ begin, objectCount });
     }
 }
 
-void Heap::refillMediumBumpRangeCache(std::lock_guard<StaticMutex>& lock, size_t sizeClass, BumpRangeCache& rangeCache)
+void Heap::allocateMediumBumpRanges(std::lock_guard<StaticMutex>& lock, size_t sizeClass, BumpAllocator& allocator, BumpRangeCache& rangeCache)
 {
     MediumPage* page = allocateMediumPage(lock, sizeClass);
     BASSERT(!rangeCache.size());
@@ -172,6 +182,12 @@ void Heap::refillMediumBumpRangeCache(std::lock_guard<StaticMutex>& lock, size_t
         if (lines[lineNumber].refCount(lock))
             continue;
 
+        // In a fragmented page, some free ranges might not fit in the cache.
+        if (rangeCache.size() == rangeCache.capacity()) {
+            m_mediumPagesWithFreeLines[sizeClass].push(page);
+            return;
+        }
+
         LineMetadata& lineMetadata = m_mediumLineMetadata[sizeClass][lineNumber];
         char* begin = lines[lineNumber].begin() + lineMetadata.startOffset;
         unsigned short objectCount = lineMetadata.objectCount;
@@ -189,7 +205,10 @@ void Heap::refillMediumBumpRangeCache(std::lock_guard<StaticMutex>& lock, size_t
             page->ref(lock);
         }
 
-        rangeCache.push({ begin, objectCount });
+        if (!allocator.canAllocate())
+            allocator.refill({ begin, objectCount });
+        else
+            rangeCache.push({ begin, objectCount });
     }
 }
 

Source/bmalloc/bmalloc/Heap.h

@@ -45,6 +45,7 @@
 namespace bmalloc {
 
 class BeginTag;
+class BumpAllocator;
 class EndTag;
 
 class Heap {
@@ -53,10 +54,10 @@ class Heap {
 
     Environment& environment() { return m_environment; }
 
-    void refillSmallBumpRangeCache(std::lock_guard<StaticMutex>&, size_t sizeClass, BumpRangeCache&);
+    void allocateSmallBumpRanges(std::lock_guard<StaticMutex>&, size_t sizeClass, BumpAllocator&, BumpRangeCache&);
     void derefSmallLine(std::lock_guard<StaticMutex>&, SmallLine*);
 
-    void refillMediumBumpRangeCache(std::lock_guard<StaticMutex>&, size_t sizeClass, BumpRangeCache&);
+    void allocateMediumBumpRanges(std::lock_guard<StaticMutex>&, size_t sizeClass, BumpAllocator&, BumpRangeCache&);
     void derefMediumLine(std::lock_guard<StaticMutex>&, MediumLine*);
 
     void* allocateLarge(std::lock_guard<StaticMutex>&, size_t);
@@ -108,11 +109,11 @@ class Heap {
     Vector<Range> m_xLargeObjects;
 
     bool m_isAllocatingPages;
+    AsyncTask<Heap, decltype(&Heap::concurrentScavenge)> m_scavenger;
 
     Environment m_environment;
 
     VMHeap m_vmHeap;
-    AsyncTask<Heap, decltype(&Heap::concurrentScavenge)> m_scavenger;
 };
 
 inline void Heap::derefSmallLine(std::lock_guard<StaticMutex>& lock, SmallLine* line)

Source/bmalloc/bmalloc/Sizes.h

@@ -93,7 +93,7 @@ namespace Sizes {
     static const uintptr_t smallOrMediumSmallTypeMask = smallType ^ mediumType; // Only valid if object is known to be small or medium.
 
     static const size_t deallocatorLogCapacity = 256;
-    static const size_t bumpRangeCacheCapacity = vmPageSize / smallLineSize / 2;
+    static const size_t bumpRangeCacheCapacity = 3;
 
     static const std::chrono::milliseconds scavengeSleepDuration = std::chrono::milliseconds(512);