348 changes: 24 additions & 324 deletions src/gc/gc.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ module gc.gc;
//debug = PTRCHECK; // more pointer checking
//debug = PTRCHECK2; // thorough but slow pointer checking
//debug = INVARIANT; // enable invariants
//debug = CACHE_HITRATE; // enable hit rate measure

/*************** Configuration *********************/

Expand Down Expand Up @@ -111,11 +110,6 @@ __gshared Duration maxPauseTime;
__gshared size_t numCollections;
__gshared size_t maxPoolMemory;

private
{
enum USE_CACHE = true;
}

private
{
extern (C)
Expand Down Expand Up @@ -1345,16 +1339,6 @@ immutable size_t[B_MAX] notbinsize = [ ~(16-1),~(32-1),~(64-1),~(128-1),~(256-1)

struct Gcx
{
static if (USE_CACHE){
byte *cached_pool_topAddr;
byte *cached_pool_baseAddr;
Pool *cached_pool;
debug (CACHE_HITRATE)
{
ulong cached_pool_queries;
ulong cached_pool_hits;
}
}
Treap!Root roots;
Treap!Range ranges;

Expand All @@ -1364,11 +1348,9 @@ struct Gcx
uint running;
int disabled; // turn off collections if >0

byte *minAddr; // min(baseAddr)
byte *maxAddr; // max(topAddr)

size_t npools;
Pool **pooltable;
import gc.pooltable;
@property size_t npools() pure const nothrow { return pooltable.length; }
PoolTable!Pool pooltable;

List*[B_MAX]bucket; // free list for each size

Expand Down Expand Up @@ -1408,11 +1390,6 @@ struct Gcx
pauseTime, maxPause);
}

debug(CACHE_HITRATE)
{
printf("\tGcx.Pool Cache hits: %llu\tqueries: %llu\n",cached_pool_hits,cached_pool_queries);
}

inited = 0;

for (size_t i = 0; i < npools; i++)
Expand All @@ -1421,11 +1398,7 @@ struct Gcx
pool.Dtor();
cstdlib.free(pool);
}
if (pooltable)
{
cstdlib.free(pooltable);
pooltable = null;
}
pooltable.Dtor();

roots.removeAll();
ranges.removeAll();
Expand All @@ -1441,24 +1414,7 @@ struct Gcx
if (inited)
{
//printf("Gcx.invariant(): this = %p\n", &this);

for (size_t i = 0; i < npools; i++)
{ auto pool = pooltable[i];

pool.Invariant();
if (i == 0)
{
assert(minAddr == pool.baseAddr);
}
if (i + 1 < npools)
{
assert(pool.opCmp(pooltable[i + 1]) < 0);
}
else if (i + 1 == npools)
{
assert(maxAddr == pool.topAddr);
}
}
pooltable.Invariant();

foreach (range; ranges)
{
Expand Down Expand Up @@ -1627,60 +1583,11 @@ struct Gcx
}
}


/**
* Find Pool that pointer is in.
* Return null if not in a Pool.
* Assume pooltable[] is sorted.
*/
Pool *findPool(bool bypassCache = !USE_CACHE)(void *p) nothrow
Pool* findPool(void* p) pure nothrow
{
static if (!bypassCache && USE_CACHE)
{
debug (CACHE_HITRATE) cached_pool_queries++;
if (p < cached_pool_topAddr
&& p >= cached_pool_baseAddr)
{
debug (CACHE_HITRATE) cached_pool_hits++;
return cached_pool;
}
}
if (p >= minAddr && p < maxAddr)
{
if (npools <= 1)
{
return npools == 0 ? null : pooltable[0];
}

/* The pooltable[] is sorted by address, so do a binary search
*/
auto pt = pooltable;
size_t low = 0;
size_t high = npools - 1;
while (low <= high)
{
size_t mid = (low + high) >> 1;
auto pool = pt[mid];
if (p < pool.baseAddr)
high = mid - 1;
else if (p >= pool.topAddr)
low = mid + 1;
else
{
static if (!bypassCache && USE_CACHE)
{
cached_pool_topAddr = pool.topAddr;
cached_pool_baseAddr = pool.baseAddr;
cached_pool = pool;
}
return pool;
}
}
}
return null;
return pooltable.findPool(p);
}


/**
* Find base address of block containing pointer p.
* Returns null if not a gc'd pointer
Expand Down Expand Up @@ -1788,15 +1695,6 @@ struct Gcx
return info;
}

void resetPoolCache() nothrow
{
static if (USE_CACHE){
cached_pool_topAddr = cached_pool_topAddr.init;
cached_pool_baseAddr = cached_pool_baseAddr.init;
cached_pool = cached_pool.init;
}
}

/**
* Compute bin for size.
*/
Expand Down Expand Up @@ -1892,181 +1790,16 @@ struct Gcx
{
debug(PRINTF) printf("Minimizing.\n");

static bool isUsed(Pool *pool) nothrow
{
return pool.freepages < pool.npages;
}

// semi-stable partition
for (size_t i = 0; i < npools; ++i)
foreach (pool; pooltable.minimize())
{
auto pool = pooltable[i];
// find first unused pool
if (isUsed(pool)) continue;

// move used pools before unused ones
size_t j = i + 1;
for (; j < npools; ++j)
{
pool = pooltable[j];
if (!isUsed(pool)) continue;
// swap
pooltable[j] = pooltable[i];
pooltable[i] = pool;
++i;
}
// npooltable[0 .. i] => used
// npooltable[i .. npools] => free

// free unused pools
for (j = i; j < npools; ++j)
{
pool = pooltable[j];
debug(PRINTF) printFreeInfo(pool);
pool.Dtor();
cstdlib.free(pool);
}
npools = i;
}

if (npools)
{
minAddr = pooltable[0].baseAddr;
maxAddr = pooltable[npools - 1].topAddr;
}
else
{
minAddr = maxAddr = null;
}

static if (USE_CACHE){
resetPoolCache();
debug(PRINTF) printFreeInfo(pool);
pool.Dtor();
cstdlib.free(pool);
}

debug(PRINTF) printf("Done minimizing.\n");
}

unittest
{
enum NPOOLS = 6;
enum NPAGES = 10;
Gcx gcx;

void reset()
{
foreach(i, ref pool; gcx.pooltable[0 .. gcx.npools])
pool.freepages = pool.npages;
gcx.minimize();
assert(gcx.npools == 0);

if (gcx.pooltable is null)
gcx.pooltable = cast(Pool**)cstdlib.malloc(NPOOLS * (Pool*).sizeof);
foreach(i; 0 .. NPOOLS)
{
auto pool = cast(Pool*)cstdlib.malloc(Pool.sizeof);
*pool = Pool.init;
gcx.pooltable[i] = pool;
}
gcx.npools = NPOOLS;
}

void usePools()
{
foreach(pool; gcx.pooltable[0 .. NPOOLS])
{
pool.pagetable = cast(ubyte*)cstdlib.malloc(NPAGES);
memset(pool.pagetable, B_FREE, NPAGES);
pool.npages = NPAGES;
pool.freepages = NPAGES / 2;
}
}

// all pools are free
reset();
assert(gcx.npools == NPOOLS);
gcx.minimize();
assert(gcx.npools == 0);

// all pools used
reset();
usePools();
assert(gcx.npools == NPOOLS);
gcx.minimize();
assert(gcx.npools == NPOOLS);

// preserves order of used pools
reset();
usePools();

{
Pool*[NPOOLS] opools = gcx.pooltable[0 .. NPOOLS];
gcx.pooltable[2].freepages = NPAGES;

gcx.minimize();
assert(gcx.npools == NPOOLS - 1);
assert(gcx.pooltable[0] == opools[0]);
assert(gcx.pooltable[1] == opools[1]);
assert(gcx.pooltable[2] == opools[3]);
}

// gcx reduces address span
reset();
usePools();

byte* base, top;

{
byte*[NPOOLS] mem = void;
foreach(i; 0 .. NPOOLS)
mem[i] = cast(byte*)os_mem_map(NPAGES * PAGESIZE);

extern(C) static int compare(in void* p1, in void *p2)
{
return p1 < p2 ? -1 : cast(int)(p2 > p1);
}
cstdlib.qsort(mem.ptr, mem.length, (byte*).sizeof, &compare);

foreach(i, pool; gcx.pooltable[0 .. NPOOLS])
{
pool.baseAddr = mem[i];
pool.topAddr = pool.baseAddr + NPAGES * PAGESIZE;
}

base = gcx.pooltable[0].baseAddr;
top = gcx.pooltable[NPOOLS - 1].topAddr;
}

gcx.minimize();
assert(gcx.npools == NPOOLS);
assert(gcx.minAddr == base);
assert(gcx.maxAddr == top);

gcx.pooltable[NPOOLS - 1].freepages = NPAGES;
gcx.pooltable[NPOOLS - 2].freepages = NPAGES;

gcx.minimize();
assert(gcx.npools == NPOOLS - 2);
assert(gcx.minAddr == base);
assert(gcx.maxAddr == gcx.pooltable[NPOOLS - 3].topAddr);

gcx.pooltable[0].freepages = NPAGES;

gcx.minimize();
assert(gcx.npools == NPOOLS - 3);
assert(gcx.minAddr != base);
assert(gcx.minAddr == gcx.pooltable[0].baseAddr);
assert(gcx.maxAddr == gcx.pooltable[NPOOLS - 4].topAddr);

// free all
foreach(pool; gcx.pooltable[0 .. gcx.npools])
pool.freepages = NPAGES;
gcx.minimize();
assert(gcx.npools == 0);
cstdlib.free(gcx.pooltable);
gcx.pooltable = null;
}


void* alloc(size_t size, ref size_t alloc_size, uint bits) nothrow
{
immutable bin = findBin(size);
Expand Down Expand Up @@ -2206,11 +1939,6 @@ struct Gcx
*/
Pool *newPool(size_t npages, bool isLargeObject) nothrow
{
Pool* pool;
Pool** newpooltable;
size_t newnpools;
size_t i;

//debug(PRINTF) printf("************Gcx::newPool(npages = %d)****************\n", npages);

// Minimum of POOLSIZE
Expand Down Expand Up @@ -2238,48 +1966,27 @@ struct Gcx

//printf("npages = %d\n", npages);

pool = cast(Pool *)cstdlib.calloc(1, Pool.sizeof);
auto pool = cast(Pool *)cstdlib.calloc(1, Pool.sizeof);
if (pool)
{
pool.initialize(npages, isLargeObject);
if (!pool.baseAddr)
goto Lerr;

newnpools = npools + 1;
newpooltable = cast(Pool **)cstdlib.realloc(pooltable, newnpools * (Pool *).sizeof);
if (!newpooltable)
goto Lerr;

// Sort pool into newpooltable[]
for (i = 0; i < npools; i++)
if (!pool.baseAddr || !pooltable.insert(pool))
{
if (pool.opCmp(newpooltable[i]) < 0)
break;
pool.Dtor();
cstdlib.free(pool);
return null;
}
memmove(newpooltable + i + 1, newpooltable + i, (npools - i) * (Pool *).sizeof);
newpooltable[i] = pool;

pooltable = newpooltable;
npools = newnpools;

minAddr = pooltable[0].baseAddr;
maxAddr = pooltable[npools - 1].topAddr;
}

if (GC.config.profile)
{
size_t gcmem = 0;
for(i = 0; i < npools; i++)
foreach (i; 0 .. npools)
gcmem += pooltable[i].topAddr - pooltable[i].baseAddr;
if(gcmem > maxPoolMemory)
maxPoolMemory = gcmem;
}
return pool;

Lerr:
pool.Dtor();
cstdlib.free(pool);
return null;
}


Expand Down Expand Up @@ -2407,12 +2114,12 @@ struct Gcx
auto p = cast(byte *)(*p1);

//if (log) debug(PRINTF) printf("\tmark %p\n", p);
if (p >= minAddr && p < maxAddr)
if (p >= pooltable.minAddr && p < pooltable.maxAddr)
{
if ((cast(size_t)p & ~cast(size_t)(PAGESIZE-1)) == pcache)
continue;

auto pool = findPool!true(p);
auto pool = findPool(p);
if (pool)
{
size_t offset = cast(size_t)(p - pool.baseAddr);
Expand Down Expand Up @@ -2855,7 +2562,7 @@ struct Gcx
{
// first, we find the Pool this block is in, then check to see if the
// mark bit is clear.
auto pool = findPool!true(addr);
auto pool = findPool(addr);
if(pool)
{
auto offset = cast(size_t)(addr - pool.baseAddr);
Expand Down Expand Up @@ -3090,7 +2797,7 @@ struct Gcx
for (size_t i = 0; i < current.dim; i++)
{
void* p = current.data[i].p;
if (!findPool!true(current.data[i].parent))
if (!findPool(current.data[i].parent))
{
auto j = prev.find(current.data[i].p);
debug(PRINTF) printf(j == OPFAIL ? "N" : " ");
Expand All @@ -3113,7 +2820,7 @@ struct Gcx
{
debug(PRINTF) printf("parent'ing unallocated memory %p, parent = %p\n", p, parent);
Pool *pool;
pool = findPool!true(p);
pool = findPool(p);
assert(pool);
size_t offset = cast(size_t)(p - pool.baseAddr);
size_t biti;
Expand All @@ -3140,7 +2847,6 @@ struct Gcx
}
}


/* ============================ Pool =============================== */


Expand Down Expand Up @@ -3420,15 +3126,9 @@ struct Pool
return size;
}

/**
* Used for sorting pooltable[]
*/
int opCmp(const Pool *p2) const nothrow
@property bool isFree() const pure nothrow
{
if (baseAddr < p2.baseAddr)
return -1;
else
return cast(int)(baseAddr > p2.baseAddr);
return npages == freepages;
}
}

Expand Down
286 changes: 286 additions & 0 deletions src/gc/pooltable.d
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,286 @@
/**
* A sorted array to quickly lookup pools.
*
* Copyright: Copyright Digital Mars 2001 -.
* License: $(WEB www.boost.org/LICENSE_1_0.txt, Boost License 1.0).
* Authors: Walter Bright, David Friedman, Sean Kelly, Martin Nowak
*/
module gc.pooltable;

static import cstdlib=core.stdc.stdlib;

struct PoolTable(Pool)
{
import core.stdc.string : memmove;

nothrow:
void Dtor()
{
cstdlib.free(pools);
pools = null;
npools = 0;
}

bool insert(Pool* pool)
{
auto newpools = cast(Pool **)cstdlib.realloc(pools, (npools + 1) * pools[0].sizeof);
if (!newpools)
return false;

pools = newpools;

// Sort pool into newpooltable[]
size_t i;
for (; i < npools; ++i)
{
if (pool.baseAddr < pools[i].baseAddr)
break;
}
if (i != npools)
memmove(pools + i + 1, pools + i, (npools - i) * pools[0].sizeof);
pools[i] = pool;

++npools;

_minAddr = pools[0].baseAddr;
_maxAddr = pools[npools - 1].topAddr;

return true;
}

@property size_t length() pure const
{
return npools;
}

ref inout(Pool*) opIndex(size_t idx) inout pure
in { assert(idx < length); }
body
{
return pools[idx];
}

inout(Pool*)[] opSlice(size_t a, size_t b) inout pure
in { assert(a <= length && b <= length); }
body
{
return pools[a .. b];
}

alias opDollar = length;

/**
* Find Pool that pointer is in.
* Return null if not in a Pool.
* Assume pooltable[] is sorted.
*/
Pool *findPool(void *p) nothrow
{
if (p >= minAddr && p < maxAddr)
{
assert(npools);

// let dmd allocate a register for this.pools
auto pools = this.pools;

if (npools == 1)
return pools[0];

/* The pooltable[] is sorted by address, so do a binary search
*/
size_t low = 0;
size_t high = npools - 1;
while (low <= high)
{
size_t mid = (low + high) >> 1;
auto pool = pools[mid];
if (p < pool.baseAddr)
high = mid - 1;
else if (p >= pool.topAddr)
low = mid + 1;
else
return pool;
}
}
return null;
}

// semi-stable partition, returns right half for which pred is false
Pool*[] minimize() pure
{
static void swap(ref Pool* a, ref Pool* b)
{
auto c = a; a = b; b = c;
}

size_t i;
// find first bad entry
for (; i < npools; ++i)
if (pools[i].isFree) break;

// move good in front of bad entries
size_t j = i + 1;
for (; j < npools; ++j)
{
if (!pools[j].isFree) // keep
swap(pools[i++], pools[j]);
}
// npooltable[0 .. i] => used pools
// npooltable[i .. npools] => free pools

if (i)
{
_minAddr = pools[0].baseAddr;
_maxAddr = pools[i - 1].topAddr;
}
else
{
_minAddr = _maxAddr = null;
}

immutable len = npools;
npools = i;
// return freed pools to the caller
return pools[npools .. len];
}

debug (INVARIANT)
void Invariant() const
{
if (!npools) return;

foreach (i, pool; pools[0 .. npools - 1])
assert(pool.baseAddr < pools[i + 1].baseAddr);

assert(_minAddr == pools[0].baseAddr);
assert(_maxAddr == pools[npools - 1].topAddr);
}

@property const(byte)* minAddr() pure const { return _minAddr; }
@property const(byte)* maxAddr() pure const { return _maxAddr; }

private:
Pool** pools;
size_t npools;
byte* _minAddr, _maxAddr;
}

unittest
{
enum NPOOLS = 6;
enum NPAGES = 10;
enum PAGESIZE = 4096;

static struct MockPool
{
byte* baseAddr, topAddr;
size_t freepages, npages;
@property bool isFree() const pure nothrow { return freepages == npages; }
}
PoolTable!MockPool pooltable;

void reset()
{
foreach(ref pool; pooltable[0 .. $])
pool.freepages = pool.npages;
pooltable.minimize();
assert(pooltable.length == 0);

foreach(i; 0 .. NPOOLS)
{
auto pool = cast(MockPool*)cstdlib.malloc(MockPool.sizeof);
*pool = MockPool.init;
assert(pooltable.insert(pool));
}
}

void usePools()
{
foreach(pool; pooltable[0 .. $])
{
pool.npages = NPAGES;
pool.freepages = NPAGES / 2;
}
}

// all pools are free
reset();
assert(pooltable.length == NPOOLS);
auto freed = pooltable.minimize();
assert(freed.length == NPOOLS);
assert(pooltable.length == 0);

// all pools used
reset();
usePools();
assert(pooltable.length == NPOOLS);
freed = pooltable.minimize();
assert(freed.length == 0);
assert(pooltable.length == NPOOLS);

// preserves order of used pools
reset();
usePools();

{
MockPool*[NPOOLS] opools = pooltable[0 .. NPOOLS];
// make the 2nd pool free
pooltable[2].freepages = NPAGES;

pooltable.minimize();
assert(pooltable.length == NPOOLS - 1);
assert(pooltable[0] == opools[0]);
assert(pooltable[1] == opools[1]);
assert(pooltable[2] == opools[3]);
}

// test that PoolTable reduces min/max address span
reset();
usePools();

byte* base, top;

{
// fill with fake addresses
size_t i;
foreach(pool; pooltable[0 .. NPOOLS])
{
pool.baseAddr = cast(byte*)(i++ * NPAGES * PAGESIZE);
pool.topAddr = pool.baseAddr + NPAGES * PAGESIZE;
}
base = pooltable[0].baseAddr;
top = pooltable[NPOOLS - 1].topAddr;
}

freed = pooltable.minimize();
assert(freed.length == 0);
assert(pooltable.length == NPOOLS);
assert(pooltable.minAddr == base);
assert(pooltable.maxAddr == top);

pooltable[NPOOLS - 1].freepages = NPAGES;
pooltable[NPOOLS - 2].freepages = NPAGES;

freed = pooltable.minimize();
assert(freed.length == 2);
assert(pooltable.length == NPOOLS - 2);
assert(pooltable.minAddr == base);
assert(pooltable.maxAddr == pooltable[NPOOLS - 3].topAddr);

pooltable[0].freepages = NPAGES;

freed = pooltable.minimize();
assert(freed.length == 1);
assert(pooltable.length == NPOOLS - 3);
assert(pooltable.minAddr != base);
assert(pooltable.minAddr == pooltable[0].baseAddr);
assert(pooltable.maxAddr == pooltable[NPOOLS - 4].topAddr);

// free all
foreach(pool; pooltable[0 .. $])
pool.freepages = NPAGES;
freed = pooltable.minimize();
assert(freed.length == NPOOLS - 3);
assert(pooltable.length == 0);
pooltable.Dtor();
}