From cc2487249c26341fd9872245a172c007b18b4b54 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 11:01:36 +0200 Subject: [PATCH 01/18] Range is no longer just bot/top, use local ScanRange --- src/gc/impl/conservative/gc.d | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 8730453494..772c346d0c 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1890,6 +1890,12 @@ struct Gcx return null; } + static struct ScanRange + { + void* pbot; + void* ptop; + } + static struct ToScanStack { nothrow: @@ -1898,25 +1904,25 @@ struct Gcx void reset() { _length = 0; - os_mem_unmap(_p, _cap * Range.sizeof); + os_mem_unmap(_p, _cap * ScanRange.sizeof); _p = null; _cap = 0; } - void push(Range rng) + void push(ScanRange rng) { if (_length == _cap) grow(); _p[_length++] = rng; } - Range pop() + ScanRange pop() in { assert(!empty); } body { return _p[--_length]; } - ref inout(Range) opIndex(size_t idx) inout + ref inout(ScanRange) opIndex(size_t idx) inout in { assert(idx < _length); } body { @@ -1930,20 +1936,20 @@ struct Gcx void grow() { enum initSize = 64 * 1024; // Windows VirtualAlloc granularity - immutable ncap = _cap ? 2 * _cap : initSize / Range.sizeof; - auto p = cast(Range*)os_mem_map(ncap * Range.sizeof); + immutable ncap = _cap ? 2 * _cap : initSize / ScanRange.sizeof; + auto p = cast(ScanRange*)os_mem_map(ncap * ScanRange.sizeof); if (p is null) onOutOfMemoryErrorNoGC(); if (_p !is null) { p[0 .. _length] = _p[0 .. _length]; - os_mem_unmap(_p, _cap * Range.sizeof); + os_mem_unmap(_p, _cap * ScanRange.sizeof); } _p = p; _cap = ncap; } size_t _length; - Range* _p; + ScanRange* _p; size_t _cap; } @@ -1960,7 +1966,7 @@ struct Gcx // limit the amount of ranges added to the toscan stack enum FANOUT_LIMIT = 32; size_t stackPos; - Range[FANOUT_LIMIT] stack = void; + ScanRange[FANOUT_LIMIT] stack = void; Lagain: size_t pcache = 0; @@ -2017,7 +2023,7 @@ struct Gcx //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = Range(base, base + binsize[bin]); + stack[stackPos++] = ScanRange(base, base + binsize[bin]); if (stackPos == stack.length) break; } @@ -2039,7 +2045,7 @@ struct Gcx continue; if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE); + stack[stackPos++] = ScanRange(base, base + pool.bPageOffsets[pn] * PAGESIZE); if (stackPos == stack.length) break; } @@ -2055,7 +2061,7 @@ struct Gcx continue; if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE); + stack[stackPos++] = ScanRange(base, base + pool.bPageOffsets[pn] * PAGESIZE); if (stackPos == stack.length) break; } @@ -2069,12 +2075,12 @@ struct Gcx } } - Range next=void; + ScanRange next=void; if (p1 < p2) { // local stack is full, push it to the global stack assert(stackPos == stack.length); - toscan.push(Range(p1, p2)); + toscan.push(ScanRange(p1, p2)); // reverse order for depth-first-order traversal foreach_reverse (ref rng; stack[0 .. $ - 1]) toscan.push(rng); From f658df854f171c8a38460ac5928ba17ef2627d1a Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 12:53:14 +0200 Subject: [PATCH 02/18] do not reload pool data, check range instead of min/max --- src/gc/impl/conservative/gc.d | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 772c346d0c..fd9108d946 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1968,22 +1968,22 @@ struct Gcx size_t stackPos; ScanRange[FANOUT_LIMIT] stack = void; - Lagain: size_t pcache = 0; // let dmd allocate a register for this.pools auto pools = pooltable.pools; const highpool = pooltable.npools - 1; const minAddr = pooltable.minAddr; - const maxAddr = pooltable.maxAddr; + size_t rngAddr = pooltable.maxAddr - minAddr; + Lagain: //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); Lnext: for (; p1 < p2; p1++) { auto p = *p1; //if (log) debug(PRINTF) printf("\tmark %p\n", p); - if (p >= minAddr && p < maxAddr) + if (cast(size_t)(p - minAddr) < rngAddr) { if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache) continue; From ae859cc220f2d2f48a0378bc016c3bba4c842a38 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 12:57:20 +0200 Subject: [PATCH 03/18] do not rescan last pointer in range --- src/gc/impl/conservative/gc.d | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index fd9108d946..8507e11577 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2080,7 +2080,8 @@ struct Gcx { // local stack is full, push it to the global stack assert(stackPos == stack.length); - toscan.push(ScanRange(p1, p2)); + if (p1 + 1 < p2) + toscan.push(ScanRange(p1 + 1, p2)); // reverse order for depth-first-order traversal foreach_reverse (ref rng; stack[0 .. $ - 1]) toscan.push(rng); From 0c34dcdccb1290002aa30b0f7518160948dc0675 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 13:00:41 +0200 Subject: [PATCH 04/18] mark: avoid unnecessary check when local stack is full --- src/gc/impl/conservative/gc.d | 57 ++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 8507e11577..3b478f87c4 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1980,6 +1980,7 @@ struct Gcx //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); Lnext: for (; p1 < p2; p1++) { + LnextBody: auto p = *p1; //if (log) debug(PRINTF) printf("\tmark %p\n", p); @@ -2009,6 +2010,7 @@ struct Gcx size_t pn = offset / PAGESIZE; Bins bin = cast(Bins)pool.pagetable[pn]; void* base = void; + void* top = void; //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); @@ -2022,10 +2024,10 @@ struct Gcx base = pool.baseAddr + offsetBase; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = ScanRange(base, base + binsize[bin]); - if (stackPos == stack.length) - break; + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + top = base + binsize[bin]; + goto LaddRange; } } else if (bin == B_PAGE) @@ -2044,10 +2046,10 @@ struct Gcx if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti)) continue; - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = ScanRange(base, base + pool.bPageOffsets[pn] * PAGESIZE); - if (stackPos == stack.length) - break; + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; } } else if (bin == B_PAGEPLUS) @@ -2060,35 +2062,40 @@ struct Gcx if(pool.nointerior.nbits && pool.nointerior.test(biti)) continue; - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - stack[stackPos++] = ScanRange(base, base + pool.bPageOffsets[pn] * PAGESIZE); - if (stackPos == stack.length) - break; + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; } } else { // Don't mark bits in B_FREE pages assert(bin == B_FREE); + } + continue; + + LaddRange: + if (stackPos < stack.length) + { + stack[stackPos++] = ScanRange(base, top); continue; } + if (p1 + 1 < p2) // *p1 already scanned + toscan.push(ScanRange(p1 + 1, p2)); + // reverse order for depth-first-order traversal + foreach_reverse (ref rng; stack) + toscan.push(rng); + stackPos = 0; + // continue with last stack entry + p1 = cast(void**)base; + p2 = cast(void**)top; + goto LnextBody; // skip increment and check } } ScanRange next=void; - if (p1 < p2) - { - // local stack is full, push it to the global stack - assert(stackPos == stack.length); - if (p1 + 1 < p2) - toscan.push(ScanRange(p1 + 1, p2)); - // reverse order for depth-first-order traversal - foreach_reverse (ref rng; stack[0 .. $ - 1]) - toscan.push(rng); - stackPos = 0; - next = stack[$-1]; - } - else if (stackPos) + if (stackPos) { // pop range from local stack and recurse next = stack[--stackPos]; From 7a4a49c5192eb43615f002d62ef2afaa8f876ca3 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 13:02:44 +0200 Subject: [PATCH 05/18] mark: skip unnecessary check for empty range --- src/gc/impl/conservative/gc.d | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 3b478f87c4..0a03020e9e 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2113,7 +2113,7 @@ struct Gcx p1 = cast(void**)next.pbot; p2 = cast(void**)next.ptop; // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); - goto Lagain; + goto LnextBody; } // collection step 1: prepare freebits and mark bits From dbcf55a93f95dee29dd1a33c175130b7448c9c6a Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 13:09:27 +0200 Subject: [PATCH 06/18] gc: use constant bitmask shift values if known --- src/gc/impl/conservative/gc.d | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 0a03020e9e..df74879989 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2020,7 +2020,7 @@ struct Gcx // We don't care abou setting pointsToBase correctly // because it's ignored for small object pools anyhow. auto offsetBase = offset & notbinsize[bin]; - biti = offsetBase >> pool.shiftBy; + biti = offsetBase >> pool.shiftBySmall; base = pool.baseAddr + offsetBase; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); @@ -2034,7 +2034,7 @@ struct Gcx { auto offsetBase = offset & notbinsize[bin]; base = pool.baseAddr + offsetBase; - biti = offsetBase >> pool.shiftBy; + biti = offsetBase >> pool.shiftByLarge; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); @@ -2056,7 +2056,7 @@ struct Gcx { pn -= pool.bPageOffsets[pn]; base = pool.baseAddr + (pn * PAGESIZE); - biti = pn * (PAGESIZE >> pool.shiftBy); + biti = pn * (PAGESIZE >> pool.shiftByLarge); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); if(pool.nointerior.nbits && pool.nointerior.test(biti)) @@ -2475,7 +2475,7 @@ struct Gcx else if(bins == B_PAGEPLUS) { pn -= pool.bPageOffsets[pn]; - biti = pn * (PAGESIZE >> pool.shiftBy); + biti = pn * (PAGESIZE >> pool.shiftByLarge); } else // bins == B_FREE { @@ -2629,6 +2629,8 @@ struct Pool bool isLargeObject; + enum shiftBySmall = 4; + enum shiftByLarge = 12; uint shiftBy; // shift count for the divisor used for determining bit indices. // This tracks how far back we have to go to find the nearest B_PAGE at @@ -2648,7 +2650,7 @@ struct Pool this.isLargeObject = isLargeObject; size_t poolsize; - shiftBy = isLargeObject ? 12 : 4; + shiftBy = isLargeObject ? shiftByLarge : shiftBySmall; //debug(PRINTF) printf("Pool::Pool(%u)\n", npages); poolsize = npages * PAGESIZE; @@ -3134,7 +3136,7 @@ struct SmallObjectPool info.base = cast(void*)((cast(size_t)p) & notbinsize[bin]); info.size = binsize[bin]; offset = info.base - baseAddr; - info.attr = getBits(cast(size_t)(offset >> shiftBy)); + info.attr = getBits(cast(size_t)(offset >> shiftBySmall)); return info; } From c25a68e087e378e24addbe965759061ecfca043c Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 17 Sep 2017 13:13:58 +0200 Subject: [PATCH 07/18] simplify flow control --- src/gc/impl/conservative/gc.d | 190 +++++++++++++++++----------------- 1 file changed, 95 insertions(+), 95 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index df74879989..711f874bb9 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1984,114 +1984,114 @@ struct Gcx auto p = *p1; //if (log) debug(PRINTF) printf("\tmark %p\n", p); - if (cast(size_t)(p - minAddr) < rngAddr) - { - if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache) - continue; + if (cast(size_t)(p - minAddr) >= rngAddr) + continue; - Pool* pool = void; - size_t low = 0; - size_t high = highpool; - while (true) - { - size_t mid = (low + high) >> 1; - pool = pools[mid]; - if (p < pool.baseAddr) - high = mid - 1; - else if (p >= pool.topAddr) - low = mid + 1; - else break; - - if (low > high) - continue Lnext; - } - size_t offset = cast(size_t)(p - pool.baseAddr); - size_t biti = void; - size_t pn = offset / PAGESIZE; - Bins bin = cast(Bins)pool.pagetable[pn]; - void* base = void; - void* top = void; + if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache) + continue; + + Pool* pool = void; + size_t low = 0; + size_t high = highpool; + while (true) + { + size_t mid = (low + high) >> 1; + pool = pools[mid]; + if (p < pool.baseAddr) + high = mid - 1; + else if (p >= pool.topAddr) + low = mid + 1; + else break; + + if (low > high) + continue Lnext; + } + size_t offset = cast(size_t)(p - pool.baseAddr); + size_t biti = void; + size_t pn = offset / PAGESIZE; + Bins bin = cast(Bins)pool.pagetable[pn]; + void* base = void; + void* top = void; - //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); + //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); - // Adjust bit to be at start of allocated memory block - if (bin < B_PAGE) + // Adjust bit to be at start of allocated memory block + if (bin < B_PAGE) + { + // We don't care abou setting pointsToBase correctly + // because it's ignored for small object pools anyhow. + auto offsetBase = offset & notbinsize[bin]; + biti = offsetBase >> pool.shiftBySmall; + base = pool.baseAddr + offsetBase; + //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); + + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - // We don't care abou setting pointsToBase correctly - // because it's ignored for small object pools anyhow. - auto offsetBase = offset & notbinsize[bin]; - biti = offsetBase >> pool.shiftBySmall; - base = pool.baseAddr + offsetBase; - //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); - - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) - { - top = base + binsize[bin]; - goto LaddRange; - } + top = base + binsize[bin]; + goto LaddRange; } - else if (bin == B_PAGE) - { - auto offsetBase = offset & notbinsize[bin]; - base = pool.baseAddr + offsetBase; - biti = offsetBase >> pool.shiftByLarge; - //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); - - pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); - - // For the NO_INTERIOR attribute. This tracks whether - // the pointer is an interior pointer or points to the - // base address of a block. - bool pointsToBase = (base == sentinel_sub(p)); - if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti)) - continue; + } + else if (bin == B_PAGE) + { + auto offsetBase = offset & notbinsize[bin]; + base = pool.baseAddr + offsetBase; + biti = offsetBase >> pool.shiftByLarge; + //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); + + pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + + // For the NO_INTERIOR attribute. This tracks whether + // the pointer is an interior pointer or points to the + // base address of a block. + bool pointsToBase = (base == sentinel_sub(p)); + if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti)) + continue; - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) - { - top = base + pool.bPageOffsets[pn] * PAGESIZE; - goto LaddRange; - } - } - else if (bin == B_PAGEPLUS) + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - pn -= pool.bPageOffsets[pn]; - base = pool.baseAddr + (pn * PAGESIZE); - biti = pn * (PAGESIZE >> pool.shiftByLarge); + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; + } + } + else if (bin == B_PAGEPLUS) + { + pn -= pool.bPageOffsets[pn]; + base = pool.baseAddr + (pn * PAGESIZE); + biti = pn * (PAGESIZE >> pool.shiftByLarge); - pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); - if(pool.nointerior.nbits && pool.nointerior.test(biti)) - continue; + pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + if(pool.nointerior.nbits && pool.nointerior.test(biti)) + continue; - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) - { - top = base + pool.bPageOffsets[pn] * PAGESIZE; - goto LaddRange; - } - } - else + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { - // Don't mark bits in B_FREE pages - assert(bin == B_FREE); + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; } - continue; + } + else + { + // Don't mark bits in B_FREE pages + assert(bin == B_FREE); + } + continue; - LaddRange: - if (stackPos < stack.length) - { - stack[stackPos++] = ScanRange(base, top); - continue; - } - if (p1 + 1 < p2) // *p1 already scanned - toscan.push(ScanRange(p1 + 1, p2)); - // reverse order for depth-first-order traversal - foreach_reverse (ref rng; stack) - toscan.push(rng); - stackPos = 0; - // continue with last stack entry - p1 = cast(void**)base; - p2 = cast(void**)top; - goto LnextBody; // skip increment and check + LaddRange: + if (stackPos < stack.length) + { + stack[stackPos++] = ScanRange(base, top); + continue; } + if (p1 + 1 < p2) // *p1 already scanned + toscan.push(ScanRange(p1 + 1, p2)); + // reverse order for depth-first-order traversal + foreach_reverse (ref rng; stack) + toscan.push(rng); + stackPos = 0; + // continue with last stack entry + p1 = cast(void**)base; + p2 = cast(void**)top; + goto LnextBody; // skip increment and check } ScanRange next=void; From ae6444863c59d14d139cc6033ca99fcf55f8307a Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sat, 23 Sep 2017 18:22:55 +0200 Subject: [PATCH 08/18] use shiftBy enum, rename rngAddr to memSize --- src/gc/impl/conservative/gc.d | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 711f874bb9..f141f6978b 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1974,9 +1974,8 @@ struct Gcx auto pools = pooltable.pools; const highpool = pooltable.npools - 1; const minAddr = pooltable.minAddr; - size_t rngAddr = pooltable.maxAddr - minAddr; + size_t memSize = pooltable.maxAddr - minAddr; - Lagain: //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); Lnext: for (; p1 < p2; p1++) { @@ -1984,7 +1983,7 @@ struct Gcx auto p = *p1; //if (log) debug(PRINTF) printf("\tmark %p\n", p); - if (cast(size_t)(p - minAddr) >= rngAddr) + if (cast(size_t)(p - minAddr) >= memSize) continue; if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache) @@ -2021,7 +2020,7 @@ struct Gcx // We don't care abou setting pointsToBase correctly // because it's ignored for small object pools anyhow. auto offsetBase = offset & notbinsize[bin]; - biti = offsetBase >> pool.shiftBySmall; + biti = offsetBase >> Pool.ShiftBy.Small; base = pool.baseAddr + offsetBase; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); @@ -2035,7 +2034,7 @@ struct Gcx { auto offsetBase = offset & notbinsize[bin]; base = pool.baseAddr + offsetBase; - biti = offsetBase >> pool.shiftByLarge; + biti = offsetBase >> Pool.ShiftBy.Large; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); @@ -2057,7 +2056,7 @@ struct Gcx { pn -= pool.bPageOffsets[pn]; base = pool.baseAddr + (pn * PAGESIZE); - biti = pn * (PAGESIZE >> pool.shiftByLarge); + biti = pn * (PAGESIZE >> Pool.ShiftBy.Large); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); if(pool.nointerior.nbits && pool.nointerior.test(biti)) @@ -2475,7 +2474,7 @@ struct Gcx else if(bins == B_PAGEPLUS) { pn -= pool.bPageOffsets[pn]; - biti = pn * (PAGESIZE >> pool.shiftByLarge); + biti = pn * (PAGESIZE >> pool.ShiftBy.Large); } else // bins == B_FREE { @@ -2629,9 +2628,12 @@ struct Pool bool isLargeObject; - enum shiftBySmall = 4; - enum shiftByLarge = 12; - uint shiftBy; // shift count for the divisor used for determining bit indices. + enum ShiftBy + { + Small = 4, + Large = 12 + } + ShiftBy shiftBy; // shift count for the divisor used for determining bit indices. // This tracks how far back we have to go to find the nearest B_PAGE at // a smaller address than a B_PAGEPLUS. To save space, we use a uint. @@ -2650,7 +2652,7 @@ struct Pool this.isLargeObject = isLargeObject; size_t poolsize; - shiftBy = isLargeObject ? shiftByLarge : shiftBySmall; + shiftBy = isLargeObject ? ShiftBy.Large : ShiftBy.Small; //debug(PRINTF) printf("Pool::Pool(%u)\n", npages); poolsize = npages * PAGESIZE; @@ -3136,7 +3138,7 @@ struct SmallObjectPool info.base = cast(void*)((cast(size_t)p) & notbinsize[bin]); info.size = binsize[bin]; offset = info.base - baseAddr; - info.attr = getBits(cast(size_t)(offset >> shiftBySmall)); + info.attr = getBits(cast(size_t)(offset >> ShiftBy.Small)); return info; } From ef551d535f1f948978aba384ac2a6e448bd43c37 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sat, 23 Sep 2017 18:50:49 +0200 Subject: [PATCH 09/18] GC.mark: calc base only if necessary --- src/gc/impl/conservative/gc.d | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index f141f6978b..87f8a8d9cb 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2021,11 +2021,11 @@ struct Gcx // because it's ignored for small object pools anyhow. auto offsetBase = offset & notbinsize[bin]; biti = offsetBase >> Pool.ShiftBy.Small; - base = pool.baseAddr + offsetBase; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { + base = pool.baseAddr + offsetBase; top = base + binsize[bin]; goto LaddRange; } @@ -2055,7 +2055,6 @@ struct Gcx else if (bin == B_PAGEPLUS) { pn -= pool.bPageOffsets[pn]; - base = pool.baseAddr + (pn * PAGESIZE); biti = pn * (PAGESIZE >> Pool.ShiftBy.Large); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); @@ -2064,6 +2063,7 @@ struct Gcx if (!pool.mark.set(biti) && !pool.noscan.test(biti)) { + base = pool.baseAddr + (pn * PAGESIZE); top = base + pool.bPageOffsets[pn] * PAGESIZE; goto LaddRange; } From 7d08f30d80e1d1a492261a1e5f2aacf98f0379e8 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 11:02:59 +0200 Subject: [PATCH 10/18] GC.mark: rearrange loop to use forward jumps only --- src/gc/impl/conservative/gc.d | 196 +++++++++++++++++----------------- 1 file changed, 98 insertions(+), 98 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 87f8a8d9cb..8b1396bb1e 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1976,103 +1976,125 @@ struct Gcx const minAddr = pooltable.minAddr; size_t memSize = pooltable.maxAddr - minAddr; + void* base = void; + void* top = void; + //printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); - Lnext: for (; p1 < p2; p1++) + for (;;) { - LnextBody: auto p = *p1; //if (log) debug(PRINTF) printf("\tmark %p\n", p); - if (cast(size_t)(p - minAddr) >= memSize) - continue; - - if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache) - continue; - - Pool* pool = void; - size_t low = 0; - size_t high = highpool; - while (true) + if (cast(size_t)(p - minAddr) < memSize && + (cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) != pcache) { - size_t mid = (low + high) >> 1; - pool = pools[mid]; - if (p < pool.baseAddr) - high = mid - 1; - else if (p >= pool.topAddr) - low = mid + 1; - else break; - - if (low > high) - continue Lnext; - } - size_t offset = cast(size_t)(p - pool.baseAddr); - size_t biti = void; - size_t pn = offset / PAGESIZE; - Bins bin = cast(Bins)pool.pagetable[pn]; - void* base = void; - void* top = void; + Pool* pool = void; + size_t low = 0; + size_t high = highpool; + while (true) + { + size_t mid = (low + high) >> 1; + pool = pools[mid]; + if (p < pool.baseAddr) + high = mid - 1; + else if (p >= pool.topAddr) + low = mid + 1; + else break; + + if (low > high) + goto LnextPtr; + } + size_t offset = cast(size_t)(p - pool.baseAddr); + size_t biti = void; + size_t pn = offset / PAGESIZE; + Bins bin = cast(Bins)pool.pagetable[pn]; - //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); + //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); - // Adjust bit to be at start of allocated memory block - if (bin < B_PAGE) - { - // We don't care abou setting pointsToBase correctly - // because it's ignored for small object pools anyhow. - auto offsetBase = offset & notbinsize[bin]; - biti = offsetBase >> Pool.ShiftBy.Small; - //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); + // Adjust bit to be at start of allocated memory block + if (bin < B_PAGE) + { + // We don't care abou setting pointsToBase correctly + // because it's ignored for small object pools anyhow. + auto offsetBase = offset & notbinsize[bin]; + biti = offsetBase >> Pool.ShiftBy.Small; + //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + base = pool.baseAddr + offsetBase; + top = base + binsize[bin]; + goto LaddRange; + } + } + else if (bin == B_PAGE) { + auto offsetBase = offset & notbinsize[bin]; base = pool.baseAddr + offsetBase; - top = base + binsize[bin]; - goto LaddRange; - } - } - else if (bin == B_PAGE) - { - auto offsetBase = offset & notbinsize[bin]; - base = pool.baseAddr + offsetBase; - biti = offsetBase >> Pool.ShiftBy.Large; - //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); - - pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); - - // For the NO_INTERIOR attribute. This tracks whether - // the pointer is an interior pointer or points to the - // base address of a block. - bool pointsToBase = (base == sentinel_sub(p)); - if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti)) - continue; + biti = offsetBase >> Pool.ShiftBy.Large; + //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) - { - top = base + pool.bPageOffsets[pn] * PAGESIZE; - goto LaddRange; + pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + + // For the NO_INTERIOR attribute. This tracks whether + // the pointer is an interior pointer or points to the + // base address of a block. + bool pointsToBase = (base == sentinel_sub(p)); + if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti)) + goto LnextPtr; + + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; + } } - } - else if (bin == B_PAGEPLUS) - { - pn -= pool.bPageOffsets[pn]; - biti = pn * (PAGESIZE >> Pool.ShiftBy.Large); + else if (bin == B_PAGEPLUS) + { + pn -= pool.bPageOffsets[pn]; + biti = pn * (PAGESIZE >> Pool.ShiftBy.Large); - pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); - if(pool.nointerior.nbits && pool.nointerior.test(biti)) - continue; + pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + if(pool.nointerior.nbits && pool.nointerior.test(biti)) + goto LnextPtr; - if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + if (!pool.mark.set(biti) && !pool.noscan.test(biti)) + { + base = pool.baseAddr + (pn * PAGESIZE); + top = base + pool.bPageOffsets[pn] * PAGESIZE; + goto LaddRange; + } + } + else { - base = pool.baseAddr + (pn * PAGESIZE); - top = base + pool.bPageOffsets[pn] * PAGESIZE; - goto LaddRange; + // Don't mark bits in B_FREE pages + assert(bin == B_FREE); } } + LnextPtr: + if (++p1 < p2) + continue; + + if (stackPos) + { + // pop range from local stack and recurse + auto next = stack.ptr[--stackPos]; + p1 = cast(void**)next.pbot; + p2 = cast(void**)next.ptop; + } + else if (!toscan.empty) + { + // pop range from global stack and recurse + auto next = toscan.pop(); + p1 = cast(void**)next.pbot; + p2 = cast(void**)next.ptop; + } else { - // Don't mark bits in B_FREE pages - assert(bin == B_FREE); + // nothing more to do + break; } + // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); continue; LaddRange: @@ -2090,29 +2112,7 @@ struct Gcx // continue with last stack entry p1 = cast(void**)base; p2 = cast(void**)top; - goto LnextBody; // skip increment and check - } - - ScanRange next=void; - if (stackPos) - { - // pop range from local stack and recurse - next = stack[--stackPos]; - } - else if (!toscan.empty) - { - // pop range from global stack and recurse - next = toscan.pop(); - } - else - { - // nothing more to do - return; } - p1 = cast(void**)next.pbot; - p2 = cast(void**)next.ptop; - // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); - goto LnextBody; } // collection step 1: prepare freebits and mark bits From f3540b877f673f90361bbd7324a8ecee397a7f73 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 11:03:50 +0200 Subject: [PATCH 11/18] do not inline ScanRange.grow --- src/gc/impl/conservative/gc.d | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 8b1396bb1e..16f1756f0c 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1935,6 +1935,8 @@ struct Gcx private: void grow() { + pragma(inline, false); + enum initSize = 64 * 1024; // Windows VirtualAlloc granularity immutable ncap = _cap ? 2 * _cap : initSize / ScanRange.sizeof; auto p = cast(ScanRange*)os_mem_map(ncap * ScanRange.sizeof); From 4cc4d6c90abe1488ee50344c958c2a967eadec56 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 11:24:07 +0200 Subject: [PATCH 12/18] GC.mark: avoid copy through temporary ScanRange --- src/gc/impl/conservative/gc.d | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 16f1756f0c..98d7d845ff 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2080,7 +2080,7 @@ struct Gcx if (stackPos) { // pop range from local stack and recurse - auto next = stack.ptr[--stackPos]; + auto next = &stack[--stackPos]; p1 = cast(void**)next.pbot; p2 = cast(void**)next.ptop; } @@ -2102,7 +2102,9 @@ struct Gcx LaddRange: if (stackPos < stack.length) { - stack[stackPos++] = ScanRange(base, top); + stack[stackPos].pbot = base; + stack[stackPos].ptop = top; + stackPos++; continue; } if (p1 + 1 < p2) // *p1 already scanned From 47945ea4765d1732e9d84aecf873b1f418337ec3 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 11:42:00 +0200 Subject: [PATCH 13/18] GC.mark: do not save result of comparison in a bool, it adds extra instructions --- src/gc/impl/conservative/gc.d | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 98d7d845ff..8328fe3699 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2041,8 +2041,7 @@ struct Gcx // For the NO_INTERIOR attribute. This tracks whether // the pointer is an interior pointer or points to the // base address of a block. - bool pointsToBase = (base == sentinel_sub(p)); - if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti)) + if(base != sentinel_sub(p) && pool.nointerior.nbits && pool.nointerior.test(biti)) goto LnextPtr; if (!pool.mark.set(biti) && !pool.noscan.test(biti)) From 3e93b744f185c0ea38c84d4e8a44ef02c595d621 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 11:42:40 +0200 Subject: [PATCH 14/18] GC.mark: use known Bin size for B_PAGE --- src/gc/impl/conservative/gc.d | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 8328fe3699..5997e45ef7 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2009,7 +2009,7 @@ struct Gcx size_t offset = cast(size_t)(p - pool.baseAddr); size_t biti = void; size_t pn = offset / PAGESIZE; - Bins bin = cast(Bins)pool.pagetable[pn]; + size_t bin = pool.pagetable[pn]; // not Bins to avoid multiple size extension instructions //debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti); @@ -2031,12 +2031,12 @@ struct Gcx } else if (bin == B_PAGE) { - auto offsetBase = offset & notbinsize[bin]; - base = pool.baseAddr + offsetBase; - biti = offsetBase >> Pool.ShiftBy.Large; + auto offsetBase = offset & ~cast(size_t)(PAGESIZE-1); + biti = offset >> Pool.ShiftBy.Large; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1); + base = cast(void*)pcache; // For the NO_INTERIOR attribute. This tracks whether // the pointer is an interior pointer or points to the From e61ae3d7d6efb477532b43a444f9c3b8a32a4c62 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 12:03:52 +0200 Subject: [PATCH 15/18] GC.mark: remove redundant calculation of the same value --- src/gc/impl/conservative/gc.d | 1 - 1 file changed, 1 deletion(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 5997e45ef7..a77e35202e 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2031,7 +2031,6 @@ struct Gcx } else if (bin == B_PAGE) { - auto offsetBase = offset & ~cast(size_t)(PAGESIZE-1); biti = offset >> Pool.ShiftBy.Large; //debug(PRINTF) printf("\t\tbiti = x%x\n", biti); From 208bf4796d1dcb4b60fb87deb86cd9670c2c6625 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Sun, 24 Sep 2017 13:06:25 +0200 Subject: [PATCH 16/18] GC.mark: p1 not incremented in addRange when stack not full --- src/gc/impl/conservative/gc.d | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index a77e35202e..9d1435f8f9 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2098,20 +2098,22 @@ struct Gcx continue; LaddRange: - if (stackPos < stack.length) + if (++p1 < p2) { - stack[stackPos].pbot = base; - stack[stackPos].ptop = top; - stackPos++; - continue; + if (stackPos < stack.length) + { + stack[stackPos].pbot = base; + stack[stackPos].ptop = top; + stackPos++; + continue; + } + toscan.push(ScanRange(p1, p2)); + // reverse order for depth-first-order traversal + foreach_reverse (ref rng; stack) + toscan.push(rng); + stackPos = 0; } - if (p1 + 1 < p2) // *p1 already scanned - toscan.push(ScanRange(p1 + 1, p2)); - // reverse order for depth-first-order traversal - foreach_reverse (ref rng; stack) - toscan.push(rng); - stackPos = 0; - // continue with last stack entry + // continue with last found range p1 = cast(void**)base; p2 = cast(void**)top; } From 3de7c0eaeb3b4ae61d2d617c4efcbd8b763546c3 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Mon, 25 Sep 2017 00:41:55 +0200 Subject: [PATCH 17/18] GC.mark: protect against empty range --- src/gc/impl/conservative/gc.d | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 9d1435f8f9..835027555d 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -1962,6 +1962,9 @@ struct Gcx */ void mark(void *pbot, void *ptop) scope nothrow { + if (pbot >= ptop) + return; + void **p1 = cast(void **)pbot; void **p2 = cast(void **)ptop; From ef9c67e3fa42344606b6265c94c3d9bb007ba970 Mon Sep 17 00:00:00 2001 From: Rainer Schuetze Date: Tue, 26 Sep 2017 07:58:56 +0200 Subject: [PATCH 18/18] GC.mark: clear pcache when getting new range no idea why this is necessary but the dyaml test fails without it in commit f658df854f171c8a38460ac5928ba17ef2627d1a --- src/gc/impl/conservative/gc.d | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/gc/impl/conservative/gc.d b/src/gc/impl/conservative/gc.d index 835027555d..2692eb6e24 100644 --- a/src/gc/impl/conservative/gc.d +++ b/src/gc/impl/conservative/gc.d @@ -2098,6 +2098,7 @@ struct Gcx break; } // printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1); + pcache = 0; continue; LaddRange: @@ -2119,6 +2120,7 @@ struct Gcx // continue with last found range p1 = cast(void**)base; p2 = cast(void**)top; + pcache = 0; } }