286 changes: 138 additions & 148 deletions src/core/atomic.d
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,17 +2,12 @@
* The atomic module provides basic support for lock-free
* concurrent programming.
*
* Copyright: Copyright Sean Kelly 2005 - 2010.
* Copyright: Copyright Sean Kelly 2005 - 2016.
* License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost License 1.0)
* Authors: Sean Kelly, Alex Rønne Petersen
* Source: $(DRUNTIMESRC core/_atomic.d)
*/

/* Copyright Sean Kelly 2005 - 2010.
* Distributed under the Boost Software License, Version 1.0.
* (See accompanying file LICENSE or copy at
* http://www.boost.org/LICENSE_1_0.txt)
*/
module core.atomic;

version( D_InlineAsm_X86 )
Expand Down Expand Up @@ -52,9 +47,19 @@ version( AsmX86 )
// NOTE: Strictly speaking, the x86 supports atomic operations on
// unaligned values. However, this is far slower than the
// common case, so such behavior should be prohibited.
private bool atomicValueIsProperlyAligned(T)( size_t addr ) pure nothrow
private bool atomicValueIsProperlyAligned(T)( ref T val ) pure nothrow @nogc @trusted
{
return addr % T.sizeof == 0;
return atomicPtrIsProperlyAligned(&val);
}

private bool atomicPtrIsProperlyAligned(T)( T* ptr ) pure nothrow @nogc @safe
{
// NOTE: 32 bit x86 systems support 8 byte CAS, which only requires
// 4 byte alignment, so use size_t as the align type here.
static if( T.sizeof > size_t.sizeof )
return cast(size_t)ptr % size_t.sizeof == 0;
else
return cast(size_t)ptr % T.sizeof == 0;
}
}

Expand All @@ -71,7 +76,7 @@ version( CoreDdoc )
* Returns:
* The result of the operation.
*/
HeadUnshared!(T) atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc
HeadUnshared!(T) atomicOp(string op, T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe
if( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) )
{
return HeadUnshared!(T).init;
Expand All @@ -91,15 +96,15 @@ version( CoreDdoc )
* Returns:
* true if the store occurred, false if not.
*/
bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe
if( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) );

/// Ditto
bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe
if( is(T == class) && __traits( compiles, { *here = writeThis; } ) );

/// Ditto
bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe
if( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) );

/**
Expand All @@ -114,7 +119,7 @@ version( CoreDdoc )
* Returns:
* The value of 'val'.
*/
HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq,T)( ref const shared T val ) pure nothrow @nogc
HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq,T)( ref const shared T val ) pure nothrow @nogc @safe
{
return HeadUnshared!(T).init;
}
Expand All @@ -130,7 +135,7 @@ version( CoreDdoc )
* val = The target variable.
* newval = The value to store.
*/
void atomicStore(MemoryOrder ms = MemoryOrder.seq,T,V1)( ref shared T val, V1 newval ) pure nothrow @nogc
void atomicStore(MemoryOrder ms = MemoryOrder.seq,T,V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe
if( __traits( compiles, { val = newval; } ) )
{

Expand Down Expand Up @@ -161,28 +166,28 @@ version( CoreDdoc )
else version( AsmX86_32 )
{
// Uses specialized asm for fast fetch and add operations
private HeadUnshared!(T) atomicFetchAdd(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc
private HeadUnshared!(T) atomicFetchAdd(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe
if( T.sizeof <= 4 && V1.sizeof <= 4)
{
size_t tmp = mod; // convert all operands to size_t
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, tmp;
mov EDX, val;
}
static if (T.sizeof == 1) asm pure nothrow @nogc { lock; xadd[EDX], AL; }
else static if (T.sizeof == 2) asm pure nothrow @nogc { lock; xadd[EDX], AX; }
else static if (T.sizeof == 4) asm pure nothrow @nogc { lock; xadd[EDX], EAX; }
static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[EDX], AL; }
else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[EDX], AX; }
else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[EDX], EAX; }

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov tmp, EAX;
}

return cast(T)tmp;
}

private HeadUnshared!(T) atomicFetchSub(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc
private HeadUnshared!(T) atomicFetchSub(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe
if( T.sizeof <= 4 && V1.sizeof <= 4)
{
return atomicFetchAdd(val, -mod);
Expand All @@ -192,12 +197,7 @@ else version( AsmX86_32 )
if( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) )
in
{
// NOTE: 32 bit x86 systems support 8 byte CAS, which only requires
// 4 byte alignment, so use size_t as the align type here.
static if( T.sizeof > size_t.sizeof )
assert( atomicValueIsProperlyAligned!(size_t)( cast(size_t) &val ) );
else
assert( atomicValueIsProperlyAligned!(T)( cast(size_t) &val ) );
assert(atomicValueIsProperlyAligned(val));
}
body
{
Expand Down Expand Up @@ -239,7 +239,7 @@ else version( AsmX86_32 )
{
get = set = atomicLoad!(MemoryOrder.raw)( val );
mixin( "set " ~ op ~ " mod;" );
} while( !cas( &val, get, set ) );
} while( !casByRef( val, get, set ) );
return set;
}
else
Expand All @@ -248,33 +248,33 @@ else version( AsmX86_32 )
}
}

bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc
bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted
{
return cas(&value, ifThis, writeThis);
}

bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe
if( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) )
{
return casImpl(here, ifThis, writeThis);
}

bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe
if( is(T == class) && __traits( compiles, { *here = writeThis; } ) )
{
return casImpl(here, ifThis, writeThis);
}

bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe
if( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) )
{
return casImpl(here, ifThis, writeThis);
}

private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc
private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe
in
{
// NOTE: 32 bit x86 systems support 8 byte CAS, which only requires
// 4 byte alignment, so use size_t as the align type here.
static if( T.sizeof > size_t.sizeof )
assert( atomicValueIsProperlyAligned!(size_t)( cast(size_t) here ) );
else
assert( atomicValueIsProperlyAligned!(T)( cast(size_t) here ) );
assert( atomicPtrIsProperlyAligned( here ) );
}
body
{
Expand All @@ -284,7 +284,7 @@ else version( AsmX86_32 )
// 1 Byte CAS
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DL, writeThis;
mov AL, ifThis;
Expand All @@ -300,7 +300,7 @@ else version( AsmX86_32 )
// 2 Byte CAS
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DX, writeThis;
mov AX, ifThis;
Expand All @@ -316,7 +316,7 @@ else version( AsmX86_32 )
// 4 Byte CAS
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EDX, writeThis;
mov EAX, ifThis;
Expand All @@ -333,7 +333,7 @@ else version( AsmX86_32 )
// 8 Byte CAS on a 32-Bit Processor
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push EDI;
push EBX;
Expand Down Expand Up @@ -391,7 +391,7 @@ else version( AsmX86_32 )
}


HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc
HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @safe
if(!__traits(isFloating, T))
{
static assert( ms != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()" );
Expand All @@ -405,7 +405,7 @@ else version( AsmX86_32 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DL, 0;
mov AL, 0;
Expand All @@ -416,7 +416,7 @@ else version( AsmX86_32 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov AL, [EAX];
Expand All @@ -431,7 +431,7 @@ else version( AsmX86_32 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DX, 0;
mov AX, 0;
Expand All @@ -442,7 +442,7 @@ else version( AsmX86_32 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov AX, [EAX];
Expand All @@ -457,7 +457,7 @@ else version( AsmX86_32 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EDX, 0;
mov EAX, 0;
Expand All @@ -468,7 +468,7 @@ else version( AsmX86_32 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov EAX, [EAX];
Expand All @@ -481,7 +481,7 @@ else version( AsmX86_32 )
// 8 Byte Load on a 32-Bit Processor
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push EDI;
push EBX;
Expand All @@ -502,7 +502,7 @@ else version( AsmX86_32 )
}
}

void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc
void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe
if( __traits( compiles, { val = newval; } ) )
{
static assert( ms != MemoryOrder.acq, "invalid MemoryOrder for atomicStore()" );
Expand All @@ -516,7 +516,7 @@ else version( AsmX86_32 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov DL, newval;
Expand All @@ -526,7 +526,7 @@ else version( AsmX86_32 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov DL, newval;
Expand All @@ -542,7 +542,7 @@ else version( AsmX86_32 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov DX, newval;
Expand All @@ -552,7 +552,7 @@ else version( AsmX86_32 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov DX, newval;
Expand All @@ -568,7 +568,7 @@ else version( AsmX86_32 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov EDX, newval;
Expand All @@ -578,7 +578,7 @@ else version( AsmX86_32 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EAX, val;
mov EDX, newval;
Expand All @@ -592,7 +592,7 @@ else version( AsmX86_32 )
// 8 Byte Store on a 32-Bit Processor
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push EDI;
push EBX;
Expand All @@ -616,11 +616,11 @@ else version( AsmX86_32 )
}


void atomicFence() nothrow
void atomicFence() nothrow @nogc @safe
{
import core.cpuid;

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
naked;

Expand Down Expand Up @@ -654,39 +654,34 @@ else version( AsmX86_32 )
else version( AsmX86_64 )
{
// Uses specialized asm for fast fetch and add operations
private HeadUnshared!(T) atomicFetchAdd(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc
private HeadUnshared!(T) atomicFetchAdd(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @trusted
if( __traits(isIntegral, T) && __traits(isIntegral, V1) )
in
{
// NOTE: 32 bit x86 systems support 8 byte CAS, which only requires
// 4 byte alignment, so use size_t as the align type here.
static if( T.sizeof > size_t.sizeof )
assert( atomicValueIsProperlyAligned!(size_t)( cast(size_t) &val ) );
else
assert( atomicValueIsProperlyAligned!(T)( cast(size_t) &val ) );
assert( atomicValueIsProperlyAligned(val));
}
body
{
size_t tmp = mod; // convert all operands to size_t
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, tmp;
mov RDX, val;
}
static if (T.sizeof == 1) asm pure nothrow @nogc { lock; xadd[RDX], AL; }
else static if (T.sizeof == 2) asm pure nothrow @nogc { lock; xadd[RDX], AX; }
else static if (T.sizeof == 4) asm pure nothrow @nogc { lock; xadd[RDX], EAX; }
else static if (T.sizeof == 8) asm pure nothrow @nogc { lock; xadd[RDX], RAX; }
static if (T.sizeof == 1) asm pure nothrow @nogc @trusted { lock; xadd[RDX], AL; }
else static if (T.sizeof == 2) asm pure nothrow @nogc @trusted { lock; xadd[RDX], AX; }
else static if (T.sizeof == 4) asm pure nothrow @nogc @trusted { lock; xadd[RDX], EAX; }
else static if (T.sizeof == 8) asm pure nothrow @nogc @trusted { lock; xadd[RDX], RAX; }

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov tmp, RAX;
}

return cast(T)tmp;
}

private HeadUnshared!(T) atomicFetchSub(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc
private HeadUnshared!(T) atomicFetchSub(T, V1)( ref shared T val, V1 mod ) pure nothrow @nogc @safe
if( __traits(isIntegral, T) && __traits(isIntegral, V1) )
{
return atomicFetchAdd(val, -mod);
Expand All @@ -696,12 +691,7 @@ else version( AsmX86_64 )
if( __traits( compiles, mixin( "*cast(T*)&val" ~ op ~ "mod" ) ) )
in
{
// NOTE: 32 bit x86 systems support 8 byte CAS, which only requires
// 4 byte alignment, so use size_t as the align type here.
static if( T.sizeof > size_t.sizeof )
assert( atomicValueIsProperlyAligned!(size_t)( cast(size_t) &val ) );
else
assert( atomicValueIsProperlyAligned!(T)( cast(size_t) &val ) );
assert( atomicValueIsProperlyAligned(val));
}
body
{
Expand Down Expand Up @@ -743,7 +733,7 @@ else version( AsmX86_64 )
{
get = set = atomicLoad!(MemoryOrder.raw)( val );
mixin( "set " ~ op ~ " mod;" );
} while( !cas( &val, get, set ) );
} while( !casByRef( val, get, set ) );
return set;
}
else
Expand All @@ -753,33 +743,33 @@ else version( AsmX86_64 )
}


bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc
bool casByRef(T,V1,V2)( ref T value, V1 ifThis, V2 writeThis ) pure nothrow @nogc @trusted
{
return cas(&value, ifThis, writeThis);
}

bool cas(T,V1,V2)( shared(T)* here, const V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe
if( !is(T == class) && !is(T U : U*) && __traits( compiles, { *here = writeThis; } ) )
{
return casImpl(here, ifThis, writeThis);
}

bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const shared(V1) ifThis, shared(V2) writeThis ) pure nothrow @nogc @safe
if( is(T == class) && __traits( compiles, { *here = writeThis; } ) )
{
return casImpl(here, ifThis, writeThis);
}

bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc
bool cas(T,V1,V2)( shared(T)* here, const shared(V1)* ifThis, shared(V2)* writeThis ) pure nothrow @nogc @safe
if( is(T U : U*) && __traits( compiles, { *here = writeThis; } ) )
{
return casImpl(here, ifThis, writeThis);
}

private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc
private bool casImpl(T,V1,V2)( shared(T)* here, V1 ifThis, V2 writeThis ) pure nothrow @nogc @safe
in
{
// NOTE: 32 bit x86 systems support 8 byte CAS, which only requires
// 4 byte alignment, so use size_t as the align type here.
static if( T.sizeof > size_t.sizeof )
assert( atomicValueIsProperlyAligned!(size_t)( cast(size_t) here ) );
else
assert( atomicValueIsProperlyAligned!(T)( cast(size_t) here ) );
assert( atomicPtrIsProperlyAligned( here ) );
}
body
{
Expand All @@ -789,7 +779,7 @@ else version( AsmX86_64 )
// 1 Byte CAS
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DL, writeThis;
mov AL, ifThis;
Expand All @@ -805,7 +795,7 @@ else version( AsmX86_64 )
// 2 Byte CAS
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DX, writeThis;
mov AX, ifThis;
Expand All @@ -821,7 +811,7 @@ else version( AsmX86_64 )
// 4 Byte CAS
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EDX, writeThis;
mov EAX, ifThis;
Expand All @@ -837,7 +827,7 @@ else version( AsmX86_64 )
// 8 Byte CAS on a 64-Bit Processor
//////////////////////////////////////////////////////////////////

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RDX, writeThis;
mov RAX, ifThis;
Expand All @@ -855,7 +845,7 @@ else version( AsmX86_64 )
version(Win64){
//Windows 64 calling convention uses different registers.
//DMD appears to reverse the register order.
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push RDI;
push RBX;
Expand All @@ -881,7 +871,7 @@ else version( AsmX86_64 )

}else{

asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push RDI;
push RBX;
Expand Down Expand Up @@ -938,7 +928,7 @@ else version( AsmX86_64 )
}


HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc
HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @safe
if(!__traits(isFloating, T))
{
static assert( ms != MemoryOrder.rel, "invalid MemoryOrder for atomicLoad()" );
Expand All @@ -952,7 +942,7 @@ else version( AsmX86_64 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DL, 0;
mov AL, 0;
Expand All @@ -963,7 +953,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov AL, [RAX];
Expand All @@ -978,7 +968,7 @@ else version( AsmX86_64 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov DX, 0;
mov AX, 0;
Expand All @@ -989,7 +979,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov AX, [RAX];
Expand All @@ -1004,7 +994,7 @@ else version( AsmX86_64 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov EDX, 0;
mov EAX, 0;
Expand All @@ -1015,7 +1005,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov EAX, [RAX];
Expand All @@ -1030,7 +1020,7 @@ else version( AsmX86_64 )

static if( needsLoadBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RDX, 0;
mov RAX, 0;
Expand All @@ -1041,7 +1031,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov RAX, [RAX];
Expand All @@ -1055,7 +1045,7 @@ else version( AsmX86_64 )
//////////////////////////////////////////////////////////////////
version(Win64){
size_t[2] retVal;
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push RDI;
push RBX;
Expand All @@ -1074,7 +1064,7 @@ else version( AsmX86_64 )
}
return cast(typeof(return)) retVal;
}else{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push RDI;
push RBX;
Expand All @@ -1097,7 +1087,7 @@ else version( AsmX86_64 )
}


void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc
void atomicStore(MemoryOrder ms = MemoryOrder.seq, T, V1)( ref shared T val, V1 newval ) pure nothrow @nogc @safe
if( __traits( compiles, { val = newval; } ) )
{
static assert( ms != MemoryOrder.acq, "invalid MemoryOrder for atomicStore()" );
Expand All @@ -1111,7 +1101,7 @@ else version( AsmX86_64 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov DL, newval;
Expand All @@ -1121,7 +1111,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov DL, newval;
Expand All @@ -1137,7 +1127,7 @@ else version( AsmX86_64 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov DX, newval;
Expand All @@ -1147,7 +1137,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov DX, newval;
Expand All @@ -1163,7 +1153,7 @@ else version( AsmX86_64 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov EDX, newval;
Expand All @@ -1173,7 +1163,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov EDX, newval;
Expand All @@ -1189,7 +1179,7 @@ else version( AsmX86_64 )

static if( needsStoreBarrier!(ms) )
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov RDX, newval;
Expand All @@ -1199,7 +1189,7 @@ else version( AsmX86_64 )
}
else
{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
mov RAX, val;
mov RDX, newval;
Expand All @@ -1213,7 +1203,7 @@ else version( AsmX86_64 )
// 16 Byte Store on a 64-Bit Processor
//////////////////////////////////////////////////////////////////
version(Win64){
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push RDI;
push RBX;
Expand All @@ -1235,7 +1225,7 @@ else version( AsmX86_64 )
pop RDI;
}
}else{
asm pure nothrow @nogc
asm pure nothrow @nogc @trusted
{
push RDI;
push RBX;
Expand All @@ -1260,10 +1250,10 @@ else version( AsmX86_64 )
}


void atomicFence() nothrow @nogc
void atomicFence() nothrow @nogc @safe
{
// SSE2 is always present in 64-bit x86 chips.
asm nothrow @nogc
asm nothrow @nogc @trusted
{
naked;

Expand All @@ -1277,7 +1267,7 @@ else version( AsmX86_64 )
// floats and doubles to ints and longs, atomically loads them, then puns
// them back. This is necessary so that they get returned in floating
// point instead of integer registers.
HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc
HeadUnshared!(T) atomicLoad(MemoryOrder ms = MemoryOrder.seq, T)( ref const shared T val ) pure nothrow @nogc @trusted
if(__traits(isFloating, T))
{
static if(T.sizeof == int.sizeof)
Expand Down Expand Up @@ -1307,7 +1297,7 @@ if(__traits(isFloating, T))

version( unittest )
{
void testCAS(T)( T val ) pure nothrow
void testCAS(T)( T val ) pure nothrow @nogc @trusted
in
{
assert(val !is T.init);
Expand All @@ -1326,7 +1316,7 @@ version( unittest )
assert( atom is val, T.stringof );
}

void testLoadStore(MemoryOrder ms = MemoryOrder.seq, T)( T val = T.init + 1 ) pure nothrow
void testLoadStore(MemoryOrder ms = MemoryOrder.seq, T)( T val = T.init + 1 ) pure nothrow @nogc @trusted
{
T base = cast(T) 0;
shared(T) atom = cast(T) 0;
Expand All @@ -1341,16 +1331,14 @@ version( unittest )
}


void testType(T)( T val = T.init + 1 ) pure nothrow
void testType(T)( T val = T.init + 1 ) pure nothrow @nogc @safe
{
testCAS!(T)( val );
testLoadStore!(MemoryOrder.seq, T)( val );
testLoadStore!(MemoryOrder.raw, T)( val );
}


//@@@BUG@@@ http://d.puremagic.com/issues/show_bug.cgi?id=8081
/+pure nothrow+/ unittest
@safe pure nothrow unittest
{
testType!(bool)();

Expand All @@ -1377,6 +1365,25 @@ version( unittest )
testType!(ulong)();
}

shared(size_t) i;

atomicOp!"+="( i, cast(size_t) 1 );
assert( i == 1 );

atomicOp!"-="( i, cast(size_t) 1 );
assert( i == 0 );

shared float f = 0;
atomicOp!"+="( f, 1 );
assert( f == 1 );

shared double d = 0;
atomicOp!"+="( d, 1 );
assert( d == 1 );
}

pure nothrow unittest
{
static if (has128BitCAS)
{
struct DoubleValue
Expand Down Expand Up @@ -1413,26 +1420,9 @@ version( unittest )
assert(head.gen == 1);
assert(cast(size_t)head.next == 1);
}

shared(size_t) i;

atomicOp!"+="( i, cast(size_t) 1 );
assert( i == 1 );

atomicOp!"-="( i, cast(size_t) 1 );
assert( i == 0 );

shared float f = 0;
atomicOp!"+="( f, 1 );
assert( f == 1 );

shared double d = 0;
atomicOp!"+="( d, 1 );
assert( d == 1 );
}

//@@@BUG@@@ http://d.puremagic.com/issues/show_bug.cgi?id=8081
/+pure nothrow+/ unittest
pure nothrow unittest
{
static struct S { int val; }
auto s = shared(S)(1);
Expand Down Expand Up @@ -1495,7 +1485,7 @@ version( unittest )
}

// === atomicFetchAdd and atomicFetchSub operations ====
unittest
pure nothrow @nogc @safe unittest
{
shared ubyte u8 = 1;
shared ushort u16 = 2;
Expand All @@ -1519,7 +1509,7 @@ version( unittest )
}
}

unittest
pure nothrow @nogc @safe unittest
{
shared ubyte u8 = 1;
shared ushort u16 = 2;
Expand Down