use todo stack when traversing GC heap

- collect all live objects in a todo stack
  to be marked later

- remove outer anychanges loop and scan GCBits

- only mark todo items after current mark(p1, p2) is done
  to benefit from cache locality, prefetcher and tail recursion

src/gc/gc.d

@@ -114,14 +114,6 @@ __gshared size_t maxPoolMemory;
 private
 {
     enum USE_CACHE = true;
-
-    // The maximum number of recursions of mark() before transitioning to
-    // multiple heap traversals to avoid consuming O(D) stack space where
-    // D is the depth of the heap graph.
-    version(Win64)
-        enum MAX_MARK_RECURSIONS = 32; // stack overflow in fibers
-    else
-        enum MAX_MARK_RECURSIONS = 64;
 }
 
 private
@@ -1374,7 +1366,6 @@ struct Gcx
 
     uint noStack;       // !=0 means don't scan stack
     uint log;           // turn on logging
-    uint anychanges;
     uint inited;
     uint running;
     int disabled;       // turn off collections if >0
@@ -1444,6 +1435,7 @@ struct Gcx
 
         roots.removeAll();
         ranges.removeAll();
+        toscan.reset();
     }
 
 
@@ -2329,27 +2321,81 @@ struct Gcx
         return 1;
     }
 
-    /**
-     * Mark overload for initial mark() call.
-     */
-    void mark(void *pbot, void *ptop) nothrow
+    static struct ToScanStack
     {
-        mark(pbot, ptop, MAX_MARK_RECURSIONS);
+    nothrow:
+        @disable this(this);
+
+        void reset()
+        {
+            _length = 0;
+            os_mem_unmap(_p, _cap * Range.sizeof);
+            _p = null;
+            _cap = 0;
+        }
+
+        void push(Range rng)
+        {
+            if (_length == _cap) grow();
+            _p[_length++] = rng;
+        }
+
+        Range pop()
+        in { assert(!empty); }
+        body
+        {
+            return _p[--_length];
+        }
+
+        ref inout(Range) opIndex(size_t idx) inout
+        in { assert(idx < _length); }
+        body
+        {
+            return _p[idx];
+        }
+
+        @property size_t length() const { return _length; }
+        @property bool empty() const { return !length; }
+
+    private:
+        void grow()
+        {
+            immutable ncap = _cap ? 2 * _cap : PAGESIZE / Range.sizeof;
+            auto p = cast(Range*)os_mem_map(ncap * Range.sizeof);
+            if (p is null) onOutOfMemoryError();
+            if (_p !is null)
+            {
+                p[0 .. _length] = _p[0 .. _length];
+                os_mem_unmap(_p, _cap * Range.sizeof);
+            }
+            _p = p;
+            _cap = ncap;
+        }
+
+        size_t _length;
+        Range* _p;
+        size_t _cap;
     }
 
+    ToScanStack toscan;
+
     /**
      * Search a range of memory values and mark any pointers into the GC pool.
      */
-    void mark(void *pbot, void *ptop, int nRecurse) nothrow
+    void mark(void *pbot, void *ptop) nothrow
     {
-        //import core.stdc.stdio;printf("nRecurse = %d\n", nRecurse);
         void **p1 = cast(void **)pbot;
         void **p2 = cast(void **)ptop;
+
+    Lagain:
         size_t pcache = 0;
-        uint changes = 0;
+        // limit the amount of ranges added to the toscan stack
+        enum FANOUT_LIMIT = 32;
+        size_t fanoutPos;
+        Range[FANOUT_LIMIT] fanout = void;
 
-        //printf("marking range: %p -> %p\n", pbot, ptop);
-        for (; p1 < p2; p1++)
+        //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
+        for (; p1 < p2 && fanoutPos != fanout.length; p1++)
         {
             auto p = cast(byte *)(*p1);
 
@@ -2420,36 +2466,31 @@ struct Gcx
                         //if (log) debug(PRINTF) printf("\t\tmarking %p\n", p);
                         if (!pool.noscan.test(biti))
                         {
-                            if(nRecurse == 0) {
-                                // Then we've got a really deep heap graph.
-                                // Start marking stuff to be scanned when we
-                                // traverse the heap again next time, to save
-                                // stack space.
-                                pool.scan.set(biti);
-                                changes = 1;
-                                pool.newChanges = true;
-                            } else {
-                                // Directly recurse mark() to prevent having
-                                // to traverse the heap O(D) times where D
-                                // is the max depth of the heap graph.
-                                if (bin < B_PAGE)
-                                {
-                                    mark(base, base + binsize[bin], nRecurse - 1);
-                                }
-                                else
-                                {
-                                    auto u = pool.bPageOffsets[pn];
-                                    mark(base, base + u * PAGESIZE, nRecurse - 1);
-                                }
-                            }
+                            immutable sz = bin < B_PAGE ? binsize[bin] : pool.bPageOffsets[pn] * PAGESIZE;
+                            fanout[fanoutPos++] = Range(base, base + sz);
                         }
 
                         debug (LOGGING) log_parent(sentinel_add(pool.baseAddr + (biti << pool.shiftBy)), sentinel_add(pbot));
                     }
                 }
             }
         }
-        anychanges |= changes;
+        // The current range has a broad fanout (e.g. array), stash
+        // it away to avoid growing the toscan stack too much.
+        assert(p1 >= p2 || fanoutPos == fanout.length);
+        if (p1 < p2)
+            toscan.push(Range(p1, p2));
+        foreach_reverse (ref rng; fanout[0 .. fanoutPos])
+            toscan.push(rng);
+        if (!toscan.empty)
+        {
+            // pop range from global stack and recurse
+            auto rng = toscan.pop();
+            p1 = cast(void**)rng.pbot;
+            p2 = cast(void**)rng.ptop;
+            // printf("  pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
+            goto Lagain;
+        }
     }
 
 
@@ -2476,12 +2517,10 @@ struct Gcx
 
         thread_suspendAll();
 
-        anychanges = 0;
         for (n = 0; n < npools; n++)
         {
             pool = pooltable[n];
             pool.mark.zero();
-            pool.scan.zero();
             if(!pool.isLargeObject) pool.freebits.zero();
         }
 
@@ -2503,7 +2542,6 @@ struct Gcx
         for (n = 0; n < npools; n++)
         {
             pool = pooltable[n];
-            pool.newChanges = false;  // Some of these get set to true on stack scan.
             if(!pool.isLargeObject)
             {
                 pool.mark.copy(&pool.freebits);
@@ -2542,65 +2580,6 @@ struct Gcx
         //log--;
 
         debug(COLLECT_PRINTF) printf("\tscan heap\n");
-        int nTraversals;
-        while (anychanges)
-        {
-            //import core.stdc.stdio;  printf("nTraversals = %d\n", ++nTraversals);
-            for (n = 0; n < npools; n++)
-            {
-                pool = pooltable[n];
-                pool.oldChanges = pool.newChanges;
-                pool.newChanges = false;
-            }
-
-            debug(COLLECT_PRINTF) printf("\t\tpass\n");
-            anychanges = 0;
-            for (n = 0; n < npools; n++)
-            {
-                pool = pooltable[n];
-                if(!pool.oldChanges) continue;
-
-                auto shiftBy = pool.shiftBy;
-                auto bbase = pool.scan.base();
-                auto btop = bbase + pool.scan.nwords;
-                //printf("\t\tn = %d, bbase = %p, btop = %p\n", n, bbase, btop);
-                for (auto b = bbase; b < btop;)
-                {
-                    auto bitm = *b;
-                    if (!bitm)
-                    {   b++;
-                        continue;
-                    }
-                    *b = 0;
-
-                    auto o = pool.baseAddr + (b - bbase) * ((typeof(bitm).sizeof*8) << shiftBy);
-
-                    auto firstset = bsf(bitm);
-                    bitm >>= firstset;
-                    o += firstset << shiftBy;
-
-                    while(bitm)
-                    {
-                        auto pn = cast(size_t)(o - pool.baseAddr) / PAGESIZE;
-                        auto bin = cast(Bins)pool.pagetable[pn];
-                        if (bin < B_PAGE)
-                        {
-                            mark(o, o + binsize[bin]);
-                        }
-                        else if (bin == B_PAGE)
-                        {
-                            auto u = pool.bPageOffsets[pn];
-                            mark(o, o + u * PAGESIZE);
-                        }
-
-                        bitm >>= 1;
-                        auto nbits = bsf(bitm);
-                        bitm >>= nbits;
-                        o += (nbits + 1) << shiftBy;
-                    }
-                }
-            }
-        }
 
         thread_processGCMarks(&isMarked);
         thread_resumeAll();
@@ -3094,7 +3073,6 @@ struct Pool
     byte* baseAddr;
     byte* topAddr;
     GCBits mark;        // entries already scanned, or should not be scanned
-    GCBits scan;        // entries that need to be scanned
     GCBits freebits;    // entries that are on the free list
     GCBits finals;      // entries that need finalizer run on them
     GCBits noscan;      // entries that should not be scanned
@@ -3107,8 +3085,6 @@ struct Pool
     ubyte* pagetable;
 
     bool isLargeObject;
-    bool oldChanges;  // Whether there were changes on the last mark.
-    bool newChanges;  // Whether there were changes on the current mark.
 
     uint shiftBy;    // shift count for the divisor used for determining bit indices.
 
@@ -3151,7 +3127,6 @@ struct Pool
         auto nbits = cast(size_t)poolsize >> shiftBy;
 
         mark.alloc(nbits);
-        scan.alloc(nbits);
 
         // pagetable already keeps track of what's free for the large object
         // pool.
@@ -3207,7 +3182,6 @@ struct Pool
             cstdlib.free(bPageOffsets);
 
         mark.Dtor();
-        scan.Dtor();
         if(isLargeObject)
         {
             nointerior.Dtor();