158 changes: 79 additions & 79 deletions src/gc/gc.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@ import rt.util.container.treap;
import cstdlib = core.stdc.stdlib : calloc, free, malloc, realloc;
import core.stdc.string : memcpy, memset, memmove;
import core.bitop;
import core.sync.mutex;
import core.thread;
static import core.memory;
private alias BlkAttr = core.memory.GC.BlkAttr;
Expand Down Expand Up @@ -136,7 +135,7 @@ private

// Declared as an extern instead of importing core.exception
// to avoid inlining - see issue 13725.
void onInvalidMemoryOperationError() nothrow;
void onInvalidMemoryOperationError() @nogc nothrow;
void onOutOfMemoryErrorNoGC() @nogc nothrow;
}

Expand Down Expand Up @@ -256,20 +255,6 @@ debug (LOGGING)
const uint GCVERSION = 1; // increment every time we change interface
// to GC.

// This just makes Mutex final to de-virtualize member function calls.
final class GCMutex : Mutex
{
final override void lock() nothrow @trusted @nogc
{
super.lock_nothrow();
}

final override void unlock() nothrow @trusted @nogc
{
super.unlock_nothrow();
}
}

struct GC
{
// For passing to debug code (not thread safe)
Expand All @@ -280,20 +265,24 @@ struct GC

Gcx *gcx; // implementation

// We can't allocate a Mutex on the GC heap because we are the GC.
// Store it in the static data segment instead.
__gshared GCMutex gcLock; // global lock
__gshared void[__traits(classInstanceSize, GCMutex)] mutexStorage;
import core.internal.spinlock;
static gcLock = shared(AlignedSpinLock)(SpinLock.Contention.lengthy);
static bool inFinalizer;

// lock GC, throw InvalidMemoryOperationError on recursive locking during finalization
static void lockNR() @nogc nothrow
{
if (inFinalizer)
onInvalidMemoryOperationError();
gcLock.lock();
}

__gshared Config config;

void initialize()
{
config.initialize();

mutexStorage[] = typeid(GCMutex).initializer[];
gcLock = cast(GCMutex) mutexStorage.ptr;
gcLock.__ctor();
gcx = cast(Gcx*)cstdlib.calloc(1, Gcx.sizeof);
if (!gcx)
onOutOfMemoryErrorNoGC();
Expand Down Expand Up @@ -328,10 +317,12 @@ struct GC
*/
void enable()
{
gcLock.lock();
scope(exit) gcLock.unlock();
assert(gcx.disabled > 0);
gcx.disabled--;
static void go(Gcx* gcx) nothrow
{
assert(gcx.disabled > 0);
gcx.disabled--;
}
runLocked!(go, otherTime, numOthers)(gcx);
}


Expand All @@ -340,15 +331,38 @@ struct GC
*/
void disable()
{
gcLock.lock();
scope(exit) gcLock.unlock();
gcx.disabled++;
static void go(Gcx* gcx) nothrow
{
gcx.disabled++;
}
runLocked!(go, otherTime, numOthers)(gcx);
}

auto runLocked(alias func, Args...)(auto ref Args args)
{
debug(PROFILE_API) immutable tm = (GC.config.profile > 1 ? currTime.ticks : 0);
lockNR();
scope (failure) gcLock.unlock();
debug(PROFILE_API) immutable tm2 = (GC.config.profile > 1 ? currTime.ticks : 0);

static if (is(typeof(func(args)) == void))
func(args);
else
auto res = func(args);

debug(PROFILE_API) if (GC.config.profile > 1)
lockTime += tm2 - tm;
gcLock.unlock();

static if (!is(typeof(func(args)) == void))
return res;
}

auto runLocked(alias func, alias time, alias count, Args...)(auto ref Args args)
{
debug(PROFILE_API) immutable tm = (GC.config.profile > 1 ? currTime.ticks : 0);
gcLock.lock();
lockNR();
scope (failure) gcLock.unlock();
debug(PROFILE_API) immutable tm2 = (GC.config.profile > 1 ? currTime.ticks : 0);

static if (is(typeof(func(args)) == void))
Expand Down Expand Up @@ -494,9 +508,6 @@ struct GC
assert(gcx);
//debug(PRINTF) printf("gcx.self = %x, pthread_self() = %x\n", gcx.self, pthread_self());

if (gcx.running)
onInvalidMemoryOperationError();

auto p = gcx.alloc(size + SENTINEL_EXTRA, alloc_size, bits);
if (!p)
onOutOfMemoryErrorNoGC();
Expand Down Expand Up @@ -564,9 +575,6 @@ struct GC
//
private void *reallocNoSync(void *p, size_t size, ref uint bits, ref size_t alloc_size, const TypeInfo ti = null) nothrow
{
if (gcx.running)
onInvalidMemoryOperationError();

if (!size)
{ if (p)
{ freeNoSync(p);
Expand Down Expand Up @@ -728,9 +736,6 @@ struct GC
}
body
{
if (gcx.running)
onInvalidMemoryOperationError();

//debug(PRINTF) printf("GC::extend(p = %p, minsize = %zu, maxsize = %zu)\n", p, minsize, maxsize);
debug (SENTINEL)
{
Expand Down Expand Up @@ -787,10 +792,7 @@ struct GC
return 0;
}

gcLock.lock();
auto rc = reserveNoSync(size);
gcLock.unlock();
return rc;
return runLocked!(reserveNoSync, otherTime, numOthers)(size);
}


Expand All @@ -802,9 +804,6 @@ struct GC
assert(size != 0);
assert(gcx);

if (gcx.running)
onInvalidMemoryOperationError();

return gcx.reserve(size);
}

Expand All @@ -831,9 +830,6 @@ struct GC
debug(PRINTF) printf("Freeing %p\n", cast(size_t) p);
assert (p);

if (gcx.running)
onInvalidMemoryOperationError();

Pool* pool;
size_t pagenum;
Bins bin;
Expand Down Expand Up @@ -1103,10 +1099,11 @@ struct GC

private auto rootIterImpl(scope int delegate(ref Root) nothrow dg) nothrow
{
gcLock.lock();
auto res = gcx.roots.opApply(dg);
gcLock.unlock();
return res;
static int go(ref Treap!(Root) roots, scope int delegate(ref Root) nothrow dg) nothrow
{
return roots.opApply(dg);
}
return runLocked!(go, otherTime, numOthers)(gcx.roots, dg);
}

/**
Expand Down Expand Up @@ -1167,10 +1164,11 @@ struct GC

private auto rangeIterImpl(scope int delegate(ref Range) nothrow dg) nothrow
{
gcLock.lock();
auto res = gcx.ranges.opApply(dg);
gcLock.unlock();
return res;
static int go(ref Treap!(Range) ranges, scope int delegate(ref Range) nothrow dg) nothrow
{
return ranges.opApply(dg);
}
return runLocked!(go, otherTime, numOthers)(gcx.ranges, dg);
}

/**
Expand All @@ -1189,15 +1187,14 @@ struct GC
size_t fullCollect() nothrow
{
debug(PRINTF) printf("GC.fullCollect()\n");
size_t result;

// Since a finalizer could launch a new thread, we always need to lock
// when collecting.
static size_t go(Gcx* gcx) nothrow
{
gcLock.lock();
result = gcx.fullcollect();
gcLock.unlock();
return gcx.fullcollect();
}
immutable result = runLocked!go(gcx);

version (none)
{
Expand All @@ -1220,11 +1217,11 @@ struct GC
{
// Since a finalizer could launch a new thread, we always need to lock
// when collecting.
static size_t go(Gcx* gcx) nothrow
{
gcLock.lock();
gcx.fullcollect(true);
gcLock.unlock();
return gcx.fullcollect(true);
}
runLocked!go(gcx);
}


Expand All @@ -1233,9 +1230,11 @@ struct GC
*/
void minimize() nothrow
{
gcLock.lock();
gcx.minimize();
gcLock.unlock();
static void go(Gcx* gcx) nothrow
{
gcx.minimize();
}
runLocked!(go, otherTime, numOthers)(gcx);
}


Expand All @@ -1245,9 +1244,7 @@ struct GC
*/
void getStats(out GCStats stats) nothrow
{
gcLock.lock();
getStatsNoSync(stats);
gcLock.unlock();
return runLocked!(getStatsNoSync, otherTime, numOthers)(stats);
}


Expand Down Expand Up @@ -1372,7 +1369,6 @@ struct Gcx

bool log; // turn on logging
debug(INVARIANT) bool initialized;
bool running;
uint disabled; // turn off collections if >0

import gc.pooltable;
Expand Down Expand Up @@ -1543,6 +1539,9 @@ struct Gcx
*/
void runFinalizers(in void[] segment) nothrow
{
GC.inFinalizer = true;
scope (failure) GC.inFinalizer = false;

foreach (pool; pooltable[0 .. npools])
{
if (!pool.finals.nbits) continue;
Expand All @@ -1558,6 +1557,7 @@ struct Gcx
spool.runFinalizers(segment);
}
}
GC.inFinalizer = false;
}

Pool* findPool(void* p) pure nothrow
Expand Down Expand Up @@ -2417,10 +2417,6 @@ struct Gcx
debug(COLLECT_PRINTF) printf("Gcx.fullcollect()\n");
//printf("\tpool address range = %p .. %p\n", minAddr, maxAddr);

if (running)
onInvalidMemoryOperationError();
running = true;

thread_suspendAll();

prepare();
Expand All @@ -2447,7 +2443,13 @@ struct Gcx
start = stop;
}

immutable freedLargePages = sweep();
GC.inFinalizer = true;
size_t freedLargePages=void;
{
scope (failure) GC.inFinalizer = false;
freedLargePages = sweep();
GC.inFinalizer = false;
}

if (GC.config.profile)
{
Expand All @@ -2465,8 +2467,6 @@ struct Gcx
++numCollections;
}

running = false; // only clear on success

updateCollectThresholds();

return freedLargePages + freedSmallPages;
Expand Down
11 changes: 11 additions & 0 deletions src/gc/proxy.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,8 @@ private
void function(void*) gc_removeRoot;
void function(void*) gc_removeRange;
void function(in void[]) gc_runFinalizers;

bool function() gc_inFinalizer;
}
}

Expand Down Expand Up @@ -96,6 +98,8 @@ private
pthis.gc_removeRoot = &gc_removeRoot;
pthis.gc_removeRange = &gc_removeRange;
pthis.gc_runFinalizers = &gc_runFinalizers;

pthis.gc_inFinalizer = &gc_inFinalizer;
}
}

Expand Down Expand Up @@ -307,6 +311,13 @@ extern (C)
return proxy.gc_runFinalizers( segment );
}

bool gc_inFinalizer() nothrow
{
if( proxy is null )
return _gc.inFinalizer;
return proxy.gc_inFinalizer();
}

Proxy* gc_getProxy() nothrow
{
return &pthis;
Expand Down
11 changes: 11 additions & 0 deletions src/gcstub/gc.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,8 @@ private
extern (C) void function(void*) gc_removeRoot;
extern (C) void function(void*) gc_removeRange;
extern (C) void function(in void[]) gc_runFinalizers;

extern (C) bool function() gc_inFinalizer;
}

__gshared Proxy pthis;
Expand Down Expand Up @@ -102,6 +104,8 @@ private
pthis.gc_removeRoot = &gc_removeRoot;
pthis.gc_removeRange = &gc_removeRange;
pthis.gc_runFinalizers = &gc_runFinalizers;

pthis.gc_inFinalizer = &gc_inFinalizer;
}

__gshared void** roots = null;
Expand Down Expand Up @@ -349,6 +353,13 @@ extern (C) void gc_runFinalizers( in void[] segment )
proxy.gc_runFinalizers( segment );
}

extern (C) bool gc_inFinalizer()
{
if( proxy !is null )
return proxy.gc_inFinalizer();
return false;
}

extern (C) Proxy* gc_getProxy()
{
return &pthis;
Expand Down
3 changes: 3 additions & 0 deletions win32.mak
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,6 +268,9 @@ $(IMPDIR)\core\internal\convert.d : src\core\internal\convert.d
$(IMPDIR)\core\internal\hash.d : src\core\internal\hash.d
copy $** $@

$(IMPDIR)\core\internal\spinlock.d : src\core\internal\spinlock.d
copy $** $@

$(IMPDIR)\core\internal\string.d : src\core\internal\string.d
copy $** $@

Expand Down
3 changes: 3 additions & 0 deletions win64.mak
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,6 +276,9 @@ $(IMPDIR)\core\internal\convert.d : src\core\internal\convert.d
$(IMPDIR)\core\internal\hash.d : src\core\internal\hash.d
copy $** $@

$(IMPDIR)\core\internal\spinlock.d : src\core\internal\spinlock.d
copy $** $@

$(IMPDIR)\core\internal\string.d : src\core\internal\string.d
copy $** $@

Expand Down