Skip to content
This repository was archived by the owner on Oct 12, 2022. It is now read-only.

GC micro-optimizations #1919

Merged
merged 18 commits into from
Sep 29, 2017
Merged

GC micro-optimizations #1919

Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
cc24872
Range is no longer just bot/top, use local ScanRange
rainers Sep 17, 2017
f658df8
do not reload pool data, check range instead of min/max
rainers Sep 17, 2017
ae859cc
do not rescan last pointer in range
rainers Sep 17, 2017
0c34dcd
mark: avoid unnecessary check when local stack is full
rainers Sep 17, 2017
7a4a49c
mark: skip unnecessary check for empty range
rainers Sep 17, 2017
dbcf55a
gc: use constant bitmask shift values if known
rainers Sep 17, 2017
c25a68e
simplify flow control
rainers Sep 17, 2017
ae64448
use shiftBy enum, rename rngAddr to memSize
rainers Sep 23, 2017
ef551d5
GC.mark: calc base only if necessary
rainers Sep 23, 2017
7d08f30
GC.mark: rearrange loop to use forward jumps only
rainers Sep 24, 2017
f3540b8
do not inline ScanRange.grow
rainers Sep 24, 2017
4cc4d6c
GC.mark: avoid copy through temporary ScanRange
rainers Sep 24, 2017
47945ea
GC.mark: do not save result of comparison in a bool, it adds extra in…
rainers Sep 24, 2017
3e93b74
GC.mark: use known Bin size for B_PAGE
rainers Sep 24, 2017
e61ae3d
GC.mark: remove redundant calculation of the same value
rainers Sep 24, 2017
208bf47
GC.mark: p1 not incremented in addRange when stack not full
rainers Sep 24, 2017
3de7c0e
GC.mark: protect against empty range
rainers Sep 24, 2017
ef9c67e
GC.mark: clear pcache when getting new range
rainers Sep 26, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
185 changes: 106 additions & 79 deletions src/gc/impl/conservative/gc.d
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1890,6 +1890,12 @@ struct Gcx
return null;
}

static struct ScanRange
{
void* pbot;
void* ptop;
}

static struct ToScanStack
{
nothrow:
Expand All @@ -1898,25 +1904,25 @@ struct Gcx
void reset()
{
_length = 0;
os_mem_unmap(_p, _cap * Range.sizeof);
os_mem_unmap(_p, _cap * ScanRange.sizeof);
_p = null;
_cap = 0;
}

void push(Range rng)
void push(ScanRange rng)
{
if (_length == _cap) grow();
_p[_length++] = rng;
}

Range pop()
ScanRange pop()
in { assert(!empty); }
body
{
return _p[--_length];
}

ref inout(Range) opIndex(size_t idx) inout
ref inout(ScanRange) opIndex(size_t idx) inout
in { assert(idx < _length); }
body
{
Expand All @@ -1929,21 +1935,23 @@ struct Gcx
private:
void grow()
{
pragma(inline, false);

enum initSize = 64 * 1024; // Windows VirtualAlloc granularity
immutable ncap = _cap ? 2 * _cap : initSize / Range.sizeof;
auto p = cast(Range*)os_mem_map(ncap * Range.sizeof);
immutable ncap = _cap ? 2 * _cap : initSize / ScanRange.sizeof;
auto p = cast(ScanRange*)os_mem_map(ncap * ScanRange.sizeof);
if (p is null) onOutOfMemoryErrorNoGC();
if (_p !is null)
{
p[0 .. _length] = _p[0 .. _length];
os_mem_unmap(_p, _cap * Range.sizeof);
os_mem_unmap(_p, _cap * ScanRange.sizeof);
}
_p = p;
_cap = ncap;
}

size_t _length;
Range* _p;
ScanRange* _p;
size_t _cap;
}

Expand All @@ -1954,34 +1962,37 @@ struct Gcx
*/
void mark(void *pbot, void *ptop) scope nothrow
{
if (pbot >= ptop)
return;

void **p1 = cast(void **)pbot;
void **p2 = cast(void **)ptop;

// limit the amount of ranges added to the toscan stack
enum FANOUT_LIMIT = 32;
size_t stackPos;
Range[FANOUT_LIMIT] stack = void;
ScanRange[FANOUT_LIMIT] stack = void;

Lagain:
size_t pcache = 0;

// let dmd allocate a register for this.pools
auto pools = pooltable.pools;
const highpool = pooltable.npools - 1;
const minAddr = pooltable.minAddr;
const maxAddr = pooltable.maxAddr;
size_t memSize = pooltable.maxAddr - minAddr;

void* base = void;
void* top = void;

//printf("marking range: [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
Lnext: for (; p1 < p2; p1++)
for (;;)
{
auto p = *p1;

//if (log) debug(PRINTF) printf("\tmark %p\n", p);
if (p >= minAddr && p < maxAddr)
if (cast(size_t)(p - minAddr) < memSize &&
(cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) != pcache)
{
if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache)
continue;

Pool* pool = void;
size_t low = 0;
size_t high = highpool;
Expand All @@ -1996,13 +2007,12 @@ struct Gcx
else break;

if (low > high)
continue Lnext;
goto LnextPtr;
}
size_t offset = cast(size_t)(p - pool.baseAddr);
size_t biti = void;
size_t pn = offset / PAGESIZE;
Bins bin = cast(Bins)pool.pagetable[pn];
void* base = void;
size_t bin = pool.pagetable[pn]; // not Bins to avoid multiple size extension instructions

//debug(PRINTF) printf("\t\tfound pool %p, base=%p, pn = %zd, bin = %d, biti = x%x\n", pool, pool.baseAddr, pn, bin, biti);

Expand All @@ -2012,94 +2022,106 @@ struct Gcx
// We don't care abou setting pointsToBase correctly
// because it's ignored for small object pools anyhow.
auto offsetBase = offset & notbinsize[bin];
biti = offsetBase >> pool.shiftBy;
base = pool.baseAddr + offsetBase;
biti = offsetBase >> Pool.ShiftBy.Small;
//debug(PRINTF) printf("\t\tbiti = x%x\n", biti);

if (!pool.mark.set(biti) && !pool.noscan.test(biti)) {
stack[stackPos++] = Range(base, base + binsize[bin]);
if (stackPos == stack.length)
break;
if (!pool.mark.set(biti) && !pool.noscan.test(biti))
{
base = pool.baseAddr + offsetBase;
top = base + binsize[bin];
goto LaddRange;
}
}
else if (bin == B_PAGE)
{
auto offsetBase = offset & notbinsize[bin];
base = pool.baseAddr + offsetBase;
biti = offsetBase >> pool.shiftBy;
biti = offset >> Pool.ShiftBy.Large;
//debug(PRINTF) printf("\t\tbiti = x%x\n", biti);

pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1);
base = cast(void*)pcache;

// For the NO_INTERIOR attribute. This tracks whether
// the pointer is an interior pointer or points to the
// base address of a block.
bool pointsToBase = (base == sentinel_sub(p));
Copy link
Member

@MartinNowak MartinNowak Sep 26, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Was it necessary to remove that name?
Even dmd should be able to optimize this and deciphering base != sentinel_sub(p) isn't that trivial.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Even dmd should be able to optimize

Unfortunately, it it doesn't just use the comparison in the following if, but stores the result to a byte with sete and uses that, adding a number of gratuitious instructions.

if(!pointsToBase && pool.nointerior.nbits && pool.nointerior.test(biti))
continue;

if (!pool.mark.set(biti) && !pool.noscan.test(biti)) {
stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE);
if (stackPos == stack.length)
break;
if(base != sentinel_sub(p) && pool.nointerior.nbits && pool.nointerior.test(biti))
goto LnextPtr;

if (!pool.mark.set(biti) && !pool.noscan.test(biti))
{
top = base + pool.bPageOffsets[pn] * PAGESIZE;
goto LaddRange;
}
}
else if (bin == B_PAGEPLUS)
{
pn -= pool.bPageOffsets[pn];
base = pool.baseAddr + (pn * PAGESIZE);
biti = pn * (PAGESIZE >> pool.shiftBy);
biti = pn * (PAGESIZE >> Pool.ShiftBy.Large);

pcache = cast(size_t)p & ~cast(size_t)(PAGESIZE-1);
if(pool.nointerior.nbits && pool.nointerior.test(biti))
continue;
goto LnextPtr;

if (!pool.mark.set(biti) && !pool.noscan.test(biti)) {
stack[stackPos++] = Range(base, base + pool.bPageOffsets[pn] * PAGESIZE);
if (stackPos == stack.length)
break;
if (!pool.mark.set(biti) && !pool.noscan.test(biti))
{
base = pool.baseAddr + (pn * PAGESIZE);
top = base + pool.bPageOffsets[pn] * PAGESIZE;
goto LaddRange;
}
}
else
{
// Don't mark bits in B_FREE pages
assert(bin == B_FREE);
continue;
}
}
}
LnextPtr:
if (++p1 < p2)
continue;

Range next=void;
if (p1 < p2)
{
// local stack is full, push it to the global stack
assert(stackPos == stack.length);
toscan.push(Range(p1, p2));
// reverse order for depth-first-order traversal
foreach_reverse (ref rng; stack[0 .. $ - 1])
toscan.push(rng);
stackPos = 0;
next = stack[$-1];
}
else if (stackPos)
{
// pop range from local stack and recurse
next = stack[--stackPos];
}
else if (!toscan.empty)
{
// pop range from global stack and recurse
next = toscan.pop();
}
else
{
// nothing more to do
return;
if (stackPos)
{
// pop range from local stack and recurse
auto next = &stack[--stackPos];
p1 = cast(void**)next.pbot;
p2 = cast(void**)next.ptop;
}
else if (!toscan.empty)
{
// pop range from global stack and recurse
auto next = toscan.pop();
p1 = cast(void**)next.pbot;
p2 = cast(void**)next.ptop;
}
else
{
// nothing more to do
break;
}
// printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
pcache = 0;
continue;

LaddRange:
if (++p1 < p2)
{
if (stackPos < stack.length)
{
stack[stackPos].pbot = base;
stack[stackPos].ptop = top;
stackPos++;
continue;
}
toscan.push(ScanRange(p1, p2));
// reverse order for depth-first-order traversal
foreach_reverse (ref rng; stack)
toscan.push(rng);
stackPos = 0;
}
// continue with last found range
p1 = cast(void**)base;
p2 = cast(void**)top;
pcache = 0;
}
p1 = cast(void**)next.pbot;
p2 = cast(void**)next.ptop;
// printf(" pop [%p..%p] (%#zx)\n", p1, p2, cast(size_t)p2 - cast(size_t)p1);
goto Lagain;
}

// collection step 1: prepare freebits and mark bits
Expand Down Expand Up @@ -2461,7 +2483,7 @@ struct Gcx
else if(bins == B_PAGEPLUS)
{
pn -= pool.bPageOffsets[pn];
biti = pn * (PAGESIZE >> pool.shiftBy);
biti = pn * (PAGESIZE >> pool.ShiftBy.Large);
}
else // bins == B_FREE
{
Expand Down Expand Up @@ -2615,7 +2637,12 @@ struct Pool

bool isLargeObject;

uint shiftBy; // shift count for the divisor used for determining bit indices.
enum ShiftBy
{
Small = 4,
Large = 12
}
ShiftBy shiftBy; // shift count for the divisor used for determining bit indices.

// This tracks how far back we have to go to find the nearest B_PAGE at
// a smaller address than a B_PAGEPLUS. To save space, we use a uint.
Expand All @@ -2634,7 +2661,7 @@ struct Pool
this.isLargeObject = isLargeObject;
size_t poolsize;

shiftBy = isLargeObject ? 12 : 4;
shiftBy = isLargeObject ? ShiftBy.Large : ShiftBy.Small;

//debug(PRINTF) printf("Pool::Pool(%u)\n", npages);
poolsize = npages * PAGESIZE;
Expand Down Expand Up @@ -3120,7 +3147,7 @@ struct SmallObjectPool
info.base = cast(void*)((cast(size_t)p) & notbinsize[bin]);
info.size = binsize[bin];
offset = info.base - baseAddr;
info.attr = getBits(cast(size_t)(offset >> shiftBy));
info.attr = getBits(cast(size_t)(offset >> ShiftBy.Small));

return info;
}
Expand Down