@@ -95,11 +95,25 @@ Apple targets have historically being problematic, xcode 4.6 would miscompile th
95
95
96
96
#include <stdatomic.h>
97
97
98
+ #if defined(HOST_ARM64 )
99
+ // C11 atomics on ARM64 offers a weaker version of sequential consistent, not expected by mono atomics operations.
100
+ // C11 seq_cst on ARM64 corresponds to acquire/release semantics, but mono expects these functions to emit a full memory
101
+ // barrier preventing any kind of reordering around the atomic operation. GCC atomics on ARM64 had similar limitations,
102
+ // see comments on GCC atomics below and mono injected full memory barriers around GCC atomic functions to mitigate this.
103
+ // Since mono GCC atomics implementation ended up even stronger (full memory barrier before/after), the C11 atomics
104
+ // implementation is still a little weaker, but should correspond to the exact same semantics as implemented by JIT
105
+ // compiler for sequential consistent atomic load/store/add/exchange/cas op codes on ARM64.
106
+ #define C11_MEMORY_ORDER_SEQ_CST () atomic_thread_fence (memory_order_seq_cst)
107
+ #else
108
+ #define C11_MEMORY_ORDER_SEQ_CST ()
109
+ #endif
110
+
98
111
static inline guint8
99
112
mono_atomic_cas_u8 (volatile guint8 * dest , guint8 exch , guint8 comp )
100
113
{
101
114
g_static_assert (sizeof (atomic_char ) == sizeof (* dest ) && ATOMIC_CHAR_LOCK_FREE == 2 );
102
115
(void )atomic_compare_exchange_strong ((volatile atomic_char * )dest , (char * )& comp , exch );
116
+ C11_MEMORY_ORDER_SEQ_CST ();
103
117
return comp ;
104
118
}
105
119
@@ -108,6 +122,7 @@ mono_atomic_cas_u16 (volatile guint16 *dest, guint16 exch, guint16 comp)
108
122
{
109
123
g_static_assert (sizeof (atomic_short ) == sizeof (* dest ) && ATOMIC_SHORT_LOCK_FREE == 2 );
110
124
(void )atomic_compare_exchange_strong ((volatile atomic_short * )dest , (short * )& comp , exch );
125
+ C11_MEMORY_ORDER_SEQ_CST ();
111
126
return comp ;
112
127
}
113
128
@@ -116,6 +131,7 @@ mono_atomic_cas_i32 (volatile gint32 *dest, gint32 exch, gint32 comp)
116
131
{
117
132
g_static_assert (sizeof (atomic_int ) == sizeof (* dest ) && ATOMIC_INT_LOCK_FREE == 2 );
118
133
(void )atomic_compare_exchange_strong ((volatile atomic_int * )dest , & comp , exch );
134
+ C11_MEMORY_ORDER_SEQ_CST ();
119
135
return comp ;
120
136
}
121
137
@@ -125,21 +141,22 @@ mono_atomic_cas_i64 (volatile gint64 *dest, gint64 exch, gint64 comp)
125
141
#if SIZEOF_LONG == 8
126
142
g_static_assert (sizeof (atomic_long ) == sizeof (* dest ) && ATOMIC_LONG_LOCK_FREE == 2 );
127
143
(void )atomic_compare_exchange_strong ((volatile atomic_long * )dest , (long * )& comp , exch );
128
- return comp ;
129
144
#elif SIZEOF_LONG_LONG == 8
130
145
g_static_assert (sizeof (atomic_llong ) == sizeof (* dest ) && ATOMIC_LLONG_LOCK_FREE == 2 );
131
146
(void )atomic_compare_exchange_strong ((volatile atomic_llong * )dest , (long long * )& comp , exch );
132
- return comp ;
133
147
#else
134
148
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
135
149
#endif
150
+ C11_MEMORY_ORDER_SEQ_CST ();
151
+ return comp ;
136
152
}
137
153
138
154
static inline gpointer
139
155
mono_atomic_cas_ptr (volatile gpointer * dest , gpointer exch , gpointer comp )
140
156
{
141
157
g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
142
158
(void )atomic_compare_exchange_strong ((volatile _Atomic (gpointer ) * )dest , & comp , exch );
159
+ C11_MEMORY_ORDER_SEQ_CST ();
143
160
return comp ;
144
161
}
145
162
@@ -191,125 +208,153 @@ static inline guint8
191
208
mono_atomic_xchg_u8 (volatile guint8 * dest , guint8 exch )
192
209
{
193
210
g_static_assert (sizeof (atomic_char ) == sizeof (* dest ) && ATOMIC_CHAR_LOCK_FREE == 2 );
194
- return atomic_exchange ((volatile atomic_char * )dest , exch );
211
+ guint8 old = atomic_exchange ((volatile atomic_char * )dest , exch );
212
+ C11_MEMORY_ORDER_SEQ_CST ();
213
+ return old ;
195
214
}
196
215
197
216
static inline guint16
198
217
mono_atomic_xchg_u16 (volatile guint16 * dest , guint16 exch )
199
218
{
200
219
g_static_assert (sizeof (atomic_short ) == sizeof (* dest ) && ATOMIC_SHORT_LOCK_FREE == 2 );
201
- return atomic_exchange ((volatile atomic_short * )dest , exch );
220
+ guint16 old = atomic_exchange ((volatile atomic_short * )dest , exch );
221
+ C11_MEMORY_ORDER_SEQ_CST ();
222
+ return old ;
202
223
}
203
224
204
225
static inline gint32
205
226
mono_atomic_xchg_i32 (volatile gint32 * dest , gint32 exch )
206
227
{
207
228
g_static_assert (sizeof (atomic_int ) == sizeof (* dest ) && ATOMIC_INT_LOCK_FREE == 2 );
208
- return atomic_exchange ((volatile atomic_int * )dest , exch );
229
+ gint32 old = atomic_exchange ((volatile atomic_int * )dest , exch );
230
+ C11_MEMORY_ORDER_SEQ_CST ();
231
+ return old ;
209
232
}
210
233
211
234
static inline gint64
212
235
mono_atomic_xchg_i64 (volatile gint64 * dest , gint64 exch )
213
236
{
214
237
#if SIZEOF_LONG == 8
215
238
g_static_assert (sizeof (atomic_long ) == sizeof (* dest ) && ATOMIC_LONG_LOCK_FREE == 2 );
216
- return atomic_exchange ((volatile atomic_long * )dest , exch );
239
+ gint64 old = atomic_exchange ((volatile atomic_long * )dest , exch );
217
240
#elif SIZEOF_LONG_LONG == 8
218
241
g_static_assert (sizeof (atomic_llong ) == sizeof (* dest ) && ATOMIC_LLONG_LOCK_FREE == 2 );
219
- return atomic_exchange ((volatile atomic_llong * )dest , exch );
242
+ gint64 old = atomic_exchange ((volatile atomic_llong * )dest , exch );
220
243
#else
221
244
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
222
245
#endif
246
+ C11_MEMORY_ORDER_SEQ_CST ();
247
+ return old ;
223
248
}
224
249
225
250
static inline gpointer
226
251
mono_atomic_xchg_ptr (volatile gpointer * dest , gpointer exch )
227
252
{
228
253
g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
229
- return atomic_exchange ((volatile _Atomic (gpointer ) * )dest , exch );
254
+ gpointer old = atomic_exchange ((volatile _Atomic (gpointer ) * )dest , exch );
255
+ C11_MEMORY_ORDER_SEQ_CST ();
256
+ return old ;
230
257
}
231
258
232
259
static inline gint32
233
260
mono_atomic_fetch_add_i32 (volatile gint32 * dest , gint32 add )
234
261
{
235
262
g_static_assert (sizeof (atomic_int ) == sizeof (* dest ) && ATOMIC_INT_LOCK_FREE == 2 );
236
- return atomic_fetch_add ((volatile atomic_int * )dest , add );
263
+ gint32 old = atomic_fetch_add ((volatile atomic_int * )dest , add );
264
+ C11_MEMORY_ORDER_SEQ_CST ();
265
+ return old ;
237
266
}
238
267
239
268
static inline gint64
240
269
mono_atomic_fetch_add_i64 (volatile gint64 * dest , gint64 add )
241
270
{
242
271
#if SIZEOF_LONG == 8
243
272
g_static_assert (sizeof (atomic_long ) == sizeof (* dest ) && ATOMIC_LONG_LOCK_FREE == 2 );
244
- return atomic_fetch_add ((volatile atomic_long * )dest , add );
273
+ gint64 old = atomic_fetch_add ((volatile atomic_long * )dest , add );
245
274
#elif SIZEOF_LONG_LONG == 8
246
275
g_static_assert (sizeof (atomic_llong ) == sizeof (* dest ) && ATOMIC_LLONG_LOCK_FREE == 2 );
247
- return atomic_fetch_add ((volatile atomic_llong * )dest , add );
276
+ gint64 old = atomic_fetch_add ((volatile atomic_llong * )dest , add );
248
277
#else
249
278
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
250
279
#endif
280
+ C11_MEMORY_ORDER_SEQ_CST ();
281
+ return old ;
251
282
}
252
283
253
284
static inline gint8
254
285
mono_atomic_load_i8 (volatile gint8 * src )
255
286
{
256
287
g_static_assert (sizeof (atomic_char ) == sizeof (* src ) && ATOMIC_CHAR_LOCK_FREE == 2 );
257
- return atomic_load ((volatile atomic_char * )src );
288
+ C11_MEMORY_ORDER_SEQ_CST ();
289
+ gint8 val = atomic_load ((volatile atomic_char * )src );
290
+ return val ;
258
291
}
259
292
260
293
static inline gint16
261
294
mono_atomic_load_i16 (volatile gint16 * src )
262
295
{
263
296
g_static_assert (sizeof (atomic_short ) == sizeof (* src ) && ATOMIC_SHORT_LOCK_FREE == 2 );
264
- return atomic_load ((volatile atomic_short * )src );
297
+ C11_MEMORY_ORDER_SEQ_CST ();
298
+ gint16 val = atomic_load ((volatile atomic_short * )src );
299
+ return val ;
265
300
}
266
301
267
302
static inline gint32 mono_atomic_load_i32 (volatile gint32 * src )
268
303
{
269
304
g_static_assert (sizeof (atomic_int ) == sizeof (* src ) && ATOMIC_INT_LOCK_FREE == 2 );
270
- return atomic_load ((volatile atomic_int * )src );
305
+ C11_MEMORY_ORDER_SEQ_CST ();
306
+ gint32 val = atomic_load ((volatile atomic_int * )src );
307
+ return val ;
271
308
}
272
309
273
310
static inline gint64
274
311
mono_atomic_load_i64 (volatile gint64 * src )
275
312
{
276
313
#if SIZEOF_LONG == 8
277
314
g_static_assert (sizeof (atomic_long ) == sizeof (* src ) && ATOMIC_LONG_LOCK_FREE == 2 );
278
- return atomic_load ((volatile atomic_long * )src );
315
+ C11_MEMORY_ORDER_SEQ_CST ();
316
+ gint64 val = atomic_load ((volatile atomic_long * )src );
279
317
#elif SIZEOF_LONG_LONG == 8
280
318
g_static_assert (sizeof (atomic_llong ) == sizeof (* src ) && ATOMIC_LLONG_LOCK_FREE == 2 );
281
- return atomic_load ((volatile atomic_llong * )src );
319
+ C11_MEMORY_ORDER_SEQ_CST ();
320
+ gint64 val = atomic_load ((volatile atomic_llong * )src );
282
321
#else
283
322
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
284
323
#endif
324
+ return val ;
285
325
}
286
326
287
327
static inline gpointer
288
328
mono_atomic_load_ptr (volatile gpointer * src )
289
329
{
290
330
g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
291
- return atomic_load ((volatile _Atomic (gpointer ) * )src );
331
+ C11_MEMORY_ORDER_SEQ_CST ();
332
+ gpointer val = atomic_load ((volatile _Atomic (gpointer ) * )src );
333
+ return val ;
292
334
}
293
335
294
336
static inline void
295
337
mono_atomic_store_i8 (volatile gint8 * dst , gint8 val )
296
338
{
297
339
g_static_assert (sizeof (atomic_char ) == sizeof (* dst ) && ATOMIC_CHAR_LOCK_FREE == 2 );
298
340
atomic_store ((volatile atomic_char * )dst , val );
341
+ C11_MEMORY_ORDER_SEQ_CST ();
299
342
}
300
343
301
344
static inline void
302
345
mono_atomic_store_i16 (volatile gint16 * dst , gint16 val )
303
346
{
304
347
g_static_assert (sizeof (atomic_short ) == sizeof (* dst ) && ATOMIC_SHORT_LOCK_FREE == 2 );
305
348
atomic_store ((volatile atomic_short * )dst , val );
349
+ C11_MEMORY_ORDER_SEQ_CST ();
306
350
}
307
351
308
352
static inline void
309
353
mono_atomic_store_i32 (volatile gint32 * dst , gint32 val )
310
354
{
311
355
g_static_assert (sizeof (atomic_int ) == sizeof (* dst ) && ATOMIC_INT_LOCK_FREE == 2 );
312
356
atomic_store ((atomic_int * )dst , val );
357
+ C11_MEMORY_ORDER_SEQ_CST ();
313
358
}
314
359
315
360
static inline void
@@ -324,13 +369,15 @@ mono_atomic_store_i64 (volatile gint64 *dst, gint64 val)
324
369
#else
325
370
#error "gint64 not same size atomic_llong or atomic_long, don't define MONO_USE_STDATOMIC"
326
371
#endif
372
+ C11_MEMORY_ORDER_SEQ_CST ();
327
373
}
328
374
329
375
static inline void
330
376
mono_atomic_store_ptr (volatile gpointer * dst , gpointer val )
331
377
{
332
378
g_static_assert (ATOMIC_POINTER_LOCK_FREE == 2 );
333
379
atomic_store ((volatile _Atomic (gpointer ) * )dst , val );
380
+ C11_MEMORY_ORDER_SEQ_CST ();
334
381
}
335
382
336
383
#elif defined(MONO_USE_WIN32_ATOMIC )
0 commit comments