Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Improve ethread atomics

The ethread atomics API now also provide double word size atomics.
Double word size atomics are implemented using native atomic
instructions on x86 (when the cmpxchg8b instruction is available)
and on x86_64 (when the cmpxchg16b instruction is available). On
other hardware where 32-bit atomics or word size atomics are
available, an optimized fallback is used; otherwise, a spinlock,
or a mutex based fallback is used.

The ethread library now performs runtime tests for presence of
hardware features, such as for example SSE2 instructions, instead
of requiring this to be determined at compile time.

There are now functions implementing each atomic operation with the
following implied memory barrier semantics: none, read, write,
acquire, release, and full. Some of the operation-barrier
combinations aren't especially useful. But instead of filtering
useful ones out, and potentially miss a useful one, we implement
them all.

A much smaller set of functionality for native atomics are required
to be implemented than before. More or less only cmpxchg and a
membar macro are required to be implemented for each atomic size.
Other functions will automatically be constructed from these. It is,
of course, often wise to implement more that this if possible from a
performance perspective.
  • Loading branch information...
commit 7f19af0423934f85c74ccb75546e5e3a6b6d10e8 1 parent 4a5a758
@rickard-green rickard-green authored
Showing with 19,442 additions and 3,039 deletions.
  1. +1 −7 INSTALL.md
  2. +1 −5 configure.in
  3. +250 −184 erts/aclocal.m4
  4. +64 −0 erts/emulator/beam/erl_bif_info.c
  5. +3 −3 erts/emulator/beam/erl_process_lock.h
  6. +498 −508 erts/emulator/beam/erl_smp.h
  7. +613 −348 erts/emulator/beam/erl_threads.h
  8. +8,562 −444 erts/include/internal/ethr_atomics.h
  9. +5 −1 erts/include/internal/ethr_internal.h
  10. +17 −13 erts/include/internal/ethr_mutex.h
  11. +98 −77 erts/include/internal/ethr_optimized_fallbacks.h
  12. +89 −73 erts/include/internal/ethread.h
  13. +107 −11 erts/include/internal/ethread_header_config.h.in
  14. +86 −165 erts/include/internal/gcc/ethr_atomic.h
  15. +115 −0 erts/include/internal/gcc/ethr_dw_atomic.h
  16. +73 −0 erts/include/internal/gcc/ethr_membar.h
  17. +14 −6 erts/include/internal/gcc/ethread.h
  18. +151 −183 erts/include/internal/i386/atomic.h
  19. +278 −0 erts/include/internal/i386/ethr_dw_atomic.h
  20. +114 −0 erts/include/internal/i386/ethr_membar.h
  21. +3 −5 erts/include/internal/i386/ethread.h
  22. +4 −1 erts/include/internal/i386/rwlock.h
  23. +17 −10 erts/include/internal/i386/spinlock.h
  24. +125 −173 erts/include/internal/libatomic_ops/ethr_atomic.h
  25. +75 −0 erts/include/internal/libatomic_ops/ethr_membar.h
  26. +13 −1 erts/include/internal/libatomic_ops/ethread.h
  27. +102 −46 erts/include/internal/ppc32/atomic.h
  28. +63 −0 erts/include/internal/ppc32/ethr_membar.h
  29. +1 −4 erts/include/internal/ppc32/ethread.h
  30. +9 −6 erts/include/internal/ppc32/rwlock.h
  31. +6 −4 erts/include/internal/ppc32/spinlock.h
  32. +3 −5 erts/include/internal/pthread/ethr_event.h
  33. +30 −164 erts/include/internal/sparc32/atomic.h
  34. +115 −0 erts/include/internal/sparc32/ethr_membar.h
  35. +2 −5 erts/include/internal/sparc32/ethread.h
  36. +10 −7 erts/include/internal/sparc32/rwlock.h
  37. +7 −4 erts/include/internal/sparc32/spinlock.h
  38. +38 −98 erts/include/internal/tile/atomic.h
  39. +35 −0 erts/include/internal/tile/ethr_membar.h
  40. +2 −3 erts/include/internal/tile/ethread.h
  41. +330 −265 erts/include/internal/win/ethr_atomic.h
  42. +154 −0 erts/include/internal/win/ethr_dw_atomic.h
  43. +8 −8 erts/include/internal/win/ethr_event.h
  44. +145 −0 erts/include/internal/win/ethr_membar.h
  45. +3 −1 erts/include/internal/win/ethread.h
  46. +8 −6 erts/lib_src/Makefile.in
  47. +4,116 −163 erts/lib_src/common/ethr_atomics.c
  48. +78 −5 erts/lib_src/common/ethr_aux.c
  49. +3 −2 erts/lib_src/common/ethr_mutex.c
  50. +31 −0 erts/lib_src/pthread/ethr_x86_sse2_asm.c
  51. +101 −1 erts/lib_src/pthread/ethread.c
  52. +2,186 −0 erts/lib_src/utils/make_atomics_api
  53. +7 −15 erts/lib_src/win/ethr_event.c
  54. +21 −1 erts/lib_src/win/ethread.c
  55. +27 −5 erts/test/ethread_SUITE.erl
  56. +425 −3 erts/test/ethread_SUITE_data/ethread_tests.c
View
8 INSTALL.md
@@ -273,12 +273,6 @@ Some of the available `configure` options are:
* `--with-ssl=PATH` - Specify location of OpenSSL include and lib
* `--{with,without}-ssl` - OpenSSL (without implies that the `crypto`,
`ssh`, and `ssl` won't be built)
-* `--enable-ethread-pre-pentium4-compatibility` - Enable compatibility with
- x86 processors before pentium 4 (back to 486) in the ethread library. If
- not passed the ethread library (part of the runtime system) will use
- instructions that first appeared on the pentium 4 processor when building
- for x86. This option will be automatically enabled if required on the
- build machine.
* `--with-libatomic_ops=PATH` - Use the `libatomic_ops` library for atomic
memory accesses. If `configure` should inform you about no native atomic
implementation available, you typically want to try using the
@@ -726,7 +720,7 @@ Copyright and License
%CopyrightBegin%
-Copyright Ericsson AB 1998-2010. All Rights Reserved.
+Copyright Ericsson AB 1998-2011. All Rights Reserved.
The contents of this file are subject to the Erlang Public License,
Version 1.1, (the "License"); you may not use this file except in
View
6 configure.in
@@ -2,7 +2,7 @@ dnl Process this file with autoconf to produce a configure script.
dnl %CopyrightBegin%
dnl
-dnl Copyright Ericsson AB 1998-2010. All Rights Reserved.
+dnl Copyright Ericsson AB 1998-2011. All Rights Reserved.
dnl
dnl The contents of this file are subject to the Erlang Public License,
dnl Version 1.1, (the "License"); you may not use this file except in
@@ -306,10 +306,6 @@ AS_HELP_STRING([--enable-m32-build],
esac
],enable_m32_build=no)
-AC_ARG_ENABLE(ethread-pre-pentium4-compatibility,
- AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility],
- [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]))
-
AC_ARG_WITH(libatomic_ops,
AS_HELP_STRING([--with-libatomic_ops=PATH],
[specify and prefer usage of libatomic_ops in the ethread library]))
View
434 erts/aclocal.m4
@@ -679,6 +679,55 @@ AC_SUBST(ERTS_INTERNAL_X_LIBS)
])
+AC_DEFUN(ETHR_CHK_SYNC_OP,
+[
+ AC_MSG_CHECKING([for $3-bit $1()])
+ case "$2" in
+ "1") sync_call="$1(&var);";;
+ "2") sync_call="$1(&var, ($4) 0);";;
+ "3") sync_call="$1(&var, ($4) 0, ($4) 0);";;
+ esac
+ have_sync_op=no
+ AC_TRY_LINK([],
+ [
+ $4 res;
+ volatile $4 var;
+ res = $sync_call
+ ],
+ [have_sync_op=yes])
+ test $have_sync_op = yes && $5
+ AC_MSG_RESULT([$have_sync_op])
+])
+
+AC_DEFUN(ETHR_CHK_INTERLOCKED,
+[
+ ilckd="$1"
+ AC_MSG_CHECKING([for ${ilckd}()])
+ case "$2" in
+ "1") ilckd_call="${ilckd}(var);";;
+ "2") ilckd_call="${ilckd}(var, ($3) 0);";;
+ "3") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0);";;
+ "4") ilckd_call="${ilckd}(var, ($3) 0, ($3) 0, arr);";;
+ esac
+ have_interlocked_op=no
+ AC_TRY_LINK(
+ [
+ #define WIN32_LEAN_AND_MEAN
+ #include <windows.h>
+ #include <intrin.h>
+ ],
+ [
+ volatile $3 *var;
+ volatile $3 arr[2];
+
+ $ilckd_call
+ return 0;
+ ],
+ [have_interlocked_op=yes])
+ test $have_interlocked_op = yes && $4
+ AC_MSG_RESULT([$have_interlocked_op])
+])
+
dnl ----------------------------------------------------------------------
dnl
dnl ERL_FIND_ETHR_LIB
@@ -750,121 +799,41 @@ case "$THR_LIB_NAME" in
AC_DEFINE(ETHR_WIN32_THREADS, 1, [Define if you have win32 threads])
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedCompareExchange64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedCompareExchange64(var, (__int64) 1, (__int64) 0);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()])
-
- AC_CHECK_SIZEOF(void *)
- case "$ac_cv_sizeof_void_p-$have_ilckd" in
- 8-no)
- ethr_have_native_atomics=no
- ethr_have_native_spinlock=no;;
- *)
- ethr_have_native_atomics=yes
- ethr_have_native_spinlock=yes;;
- esac
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedDecrement64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedDecrement64(var);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedIncrement64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedIncrement64(var);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedExchangeAdd64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedExchangeAdd64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedExchange64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedExchange64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedAnd64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedAnd64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()])
-
- have_ilckd=no
- AC_MSG_CHECKING([for _InterlockedOr64()])
- AC_TRY_LINK([
- #define WIN32_LEAN_AND_MEAN
- #include <windows.h>
- ],
- [
- volatile __int64 *var;
- _InterlockedOr64(var, (__int64) 1);
- return 0;
- ],
- have_ilckd=yes)
- AC_MSG_RESULT([$have_ilckd])
- test $have_ilckd = yes && AC_DEFINE(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()])
-
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT, 1, [Define if you have _InterlockedDecrement()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement_rel], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT_REL, 1, [Define if you have _InterlockedDecrement_rel()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT, 1, [Define if you have _InterlockedIncrement()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement_acq], [1], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT_ACQ, 1, [Define if you have _InterlockedIncrement_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD, 1, [Define if you have _InterlockedExchangeAdd()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd_acq], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD_ACQ, 1, [Define if you have _InterlockedExchangeAdd_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedAnd], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND, 1, [Define if you have _InterlockedAnd()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedOr], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR, 1, [Define if you have _InterlockedOr()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchange], [2], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE, 1, [Define if you have _InterlockedExchange()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE, 1, [Define if you have _InterlockedCompareExchange()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_acq], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_ACQ, 1, [Define if you have _InterlockedCompareExchange_acq()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange_rel], [3], [long], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE_REL, 1, [Define if you have _InterlockedCompareExchange_rel()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64, 1, [Define if you have _InterlockedDecrement64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedDecrement64_rel], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDDECREMENT64_REL, 1, [Define if you have _InterlockedDecrement64_rel()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement64], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64, 1, [Define if you have _InterlockedIncrement64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedIncrement64_acq], [1], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDINCREMENT64_ACQ, 1, [Define if you have _InterlockedIncrement64_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64, 1, [Define if you have _InterlockedExchangeAdd64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchangeAdd64_acq], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGEADD64_ACQ, 1, [Define if you have _InterlockedExchangeAdd64_acq()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedAnd64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDAND64, 1, [Define if you have _InterlockedAnd64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedOr64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDOR64, 1, [Define if you have _InterlockedOr64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedExchange64], [2], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDEXCHANGE64, 1, [Define if you have _InterlockedExchange64()]))
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64, 1, [Define if you have _InterlockedCompareExchange64()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_acq], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_ACQ, 1, [Define if you have _InterlockedCompareExchange64_acq()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange64_rel], [3], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE64_REL, 1, [Define if you have _InterlockedCompareExchange64_rel()]))
+ test "$have_interlocked_op" = "yes" && ethr_have_native_atomics=yes
+
+ ETHR_CHK_INTERLOCKED([_InterlockedCompareExchange128], [4], [__int64], AC_DEFINE_UNQUOTED(ETHR_HAVE__INTERLOCKEDCOMPAREEXCHANGE128, 1, [Define if you have _InterlockedCompareExchange128()]))
+
+ test "$ethr_have_native_atomics" = "yes" && ethr_have_native_spinlock=yes
;;
pthread)
@@ -1100,35 +1069,51 @@ case "$THR_LIB_NAME" in
AC_MSG_RESULT([$linux_futex])
test $linux_futex = yes && AC_DEFINE(ETHR_HAVE_LINUX_FUTEX, 1, [Define if you have a linux futex implementation.])
- AC_MSG_CHECKING([for GCC atomic operations])
- ethr_have_gcc_atomic_ops=no
- AC_TRY_LINK([],
- [
- long res;
- volatile long val;
- res = __sync_val_compare_and_swap(&val, (long) 1, (long) 0);
- res = __sync_add_and_fetch(&val, (long) 1);
- res = __sync_sub_and_fetch(&val, (long) 1);
- res = __sync_fetch_and_and(&val, (long) 1);
- res = __sync_fetch_and_or(&val, (long) 1);
- ],
- [ethr_have_native_atomics=yes
- ethr_have_gcc_atomic_ops=yes])
- AC_MSG_RESULT([$ethr_have_gcc_atomic_ops])
- test $ethr_have_gcc_atomic_ops = yes && AC_DEFINE(ETHR_HAVE_GCC_ATOMIC_OPS, 1, [Define if you have gcc atomic operations])
+ AC_CHECK_SIZEOF(int)
+ AC_CHECK_SIZEOF(long)
+ AC_CHECK_SIZEOF(long long)
+ AC_CHECK_SIZEOF(__int128_t)
+
+ if test "$ac_cv_sizeof_int" = "4"; then
+ int32="int"
+ elif test "$ac_cv_sizeof_long" = "4"; then
+ int32="long"
+ elif test "$ac_cv_sizeof_long_long" = "4"; then
+ int32="long long"
+ else
+ AC_MSG_ERROR([No 32-bit type found])
+ fi
- case "$host_cpu" in
- sun4u | sparc64 | sun4v)
- ethr_have_native_atomics=yes;;
- i86pc | i*86 | x86_64 | amd64)
- ethr_have_native_atomics=yes;;
- macppc | ppc | "Power Macintosh")
- ethr_have_native_atomics=yes;;
- tile)
- ethr_have_native_atomics=yes;;
- *)
- ;;
- esac
+ if test "$ac_cv_sizeof_int" = "8"; then
+ int64="int"
+ elif test "$ac_cv_sizeof_long" = "8"; then
+ int64="long"
+ elif test "$ac_cv_sizeof_long_long" = "8"; then
+ int64="long long"
+ else
+ AC_MSG_ERROR([No 64-bit type found])
+ fi
+
+ int128=no
+ if test "$ac_cv_sizeof___int128_t" = "16"; then
+ int128="__int128_t"
+ fi
+
+ ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP32, 1, [Define if you have __sync_val_compare_and_swap() for 32-bit integers]))
+ test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH32, 1, [Define if you have __sync_add_and_fetch() for 32-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND32, 1, [Define if you have __sync_fetch_and_and() for 32-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [32], [$int32], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR32, 1, [Define if you have __sync_fetch_and_or() for 32-bit integers]))
+
+ ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP64, 1, [Define if you have __sync_val_compare_and_swap() for 64-bit integers]))
+ test "$have_sync_op" = "yes" && ethr_have_native_atomics=yes
+ ETHR_CHK_SYNC_OP([__sync_add_and_fetch], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_ADD_AND_FETCH64, 1, [Define if you have __sync_add_and_fetch() for 64-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_and], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_AND64, 1, [Define if you have __sync_fetch_and_and() for 64-bit integers]))
+ ETHR_CHK_SYNC_OP([__sync_fetch_and_or], [2], [64], [$int64], AC_DEFINE(ETHR_HAVE___SYNC_FETCH_AND_OR64, 1, [Define if you have __sync_fetch_and_or() for 64-bit integers]))
+
+ if test $int128 != no; then
+ ETHR_CHK_SYNC_OP([__sync_val_compare_and_swap], [3], [128], [$int128], AC_DEFINE(ETHR_HAVE___SYNC_VAL_COMPARE_AND_SWAP128, 1, [Define if you have __sync_val_compare_and_swap() for 128-bit integers]))
+ fi
AC_MSG_CHECKING([for a usable libatomic_ops implementation])
case "x$with_libatomic_ops" in
@@ -1175,6 +1160,34 @@ case "$THR_LIB_NAME" in
AC_MSG_ERROR([No usable libatomic_ops implementation found])
fi
+ case "$host_cpu" in
+ sparc | sun4u | sparc64 | sun4v)
+ case "$with_sparc_memory_order" in
+ "TSO")
+ AC_DEFINE(ETHR_SPARC_TSO, 1, [Define if only run in Sparc TSO mode]);;
+ "PSO")
+ AC_DEFINE(ETHR_SPARC_PSO, 1, [Define if only run in Sparc PSO, or TSO mode]);;
+ "RMO"|"")
+ AC_DEFINE(ETHR_SPARC_RMO, 1, [Define if run in Sparc RMO, PSO, or TSO mode]);;
+ *)
+ AC_MSG_ERROR([Unsupported Sparc memory order: $with_sparc_memory_order]);;
+ esac
+ ethr_have_native_atomics=yes;;
+ i86pc | i*86 | x86_64 | amd64)
+ if test "$enable_x86_out_of_order" = "yes"; then
+ AC_DEFINE(ETHR_X86_OUT_OF_ORDER, 1, [Define if x86/x86_64 out of order instructions should be synchronized])
+ fi
+ ethr_have_native_atomics=yes;;
+ macppc | ppc | "Power Macintosh")
+ ethr_have_native_atomics=yes;;
+ tile)
+ ethr_have_native_atomics=yes;;
+ *)
+ ;;
+ esac
+
+ test ethr_have_native_atomics = "yes" && ethr_have_native_spinlock=yes
+
dnl Restore LIBS
LIBS=$saved_libs
dnl restore CPPFLAGS
@@ -1210,6 +1223,8 @@ AC_CHECK_SIZEOF(long long)
AC_DEFINE_UNQUOTED(ETHR_SIZEOF_LONG_LONG, $ac_cv_sizeof_long_long, [Define to the size of long long])
AC_CHECK_SIZEOF(__int64)
AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT64, $ac_cv_sizeof___int64, [Define to the size of __int64])
+AC_CHECK_SIZEOF(__int128_t)
+AC_DEFINE_UNQUOTED(ETHR_SIZEOF___INT128_T, $ac_cv_sizeof___int128_t, [Define to the size of __int128_t])
case X$erl_xcomp_bigendian in
@@ -1232,6 +1247,10 @@ AC_ARG_ENABLE(native-ethr-impls,
*) disable_native_ethr_impls=no ;;
esac ], disable_native_ethr_impls=no)
+AC_ARG_ENABLE(x86-out-of-order,
+ AS_HELP_STRING([--enable-x86-out-of-order],
+ [enable x86/x84_64 out of order support (default disabled)]))
+
test "X$disable_native_ethr_impls" = "Xyes" &&
AC_DEFINE(ETHR_DISABLE_NATIVE_IMPLS, 1, [Define if you want to disable native ethread implementations])
@@ -1250,55 +1269,101 @@ AC_ARG_WITH(libatomic_ops,
AS_HELP_STRING([--with-libatomic_ops=PATH],
[specify and prefer usage of libatomic_ops in the ethread library]))
-AC_ARG_ENABLE(ethread-pre-pentium4-compatibility,
- AS_HELP_STRING([--enable-ethread-pre-pentium4-compatibility],
- [enable compatibility with x86 processors before pentium 4 (back to 486) in the ethread library]),
-[
- case "$enable_ethread_pre_pentium4_compatibility" in
- yes|no) ;;
- *) enable_ethread_pre_pentium4_compatibility=check;;
- esac
-],
-[enable_ethread_pre_pentium4_compatibility=check])
-
-test "$cross_compiling" != "yes" || enable_ethread_pre_pentium4_compatibility=no
-
-case "$enable_ethread_pre_pentium4_compatibility-$host_cpu" in
- check-i86pc | check-i*86)
- AC_MSG_CHECKING([whether pre pentium 4 compatibility should forced])
- AC_RUN_IFELSE([
-#if defined(__GNUC__)
-# if defined(ETHR_PREFER_LIBATOMIC_OPS_NATIVE_IMPLS)
-# define CHECK_LIBATOMIC_OPS__
-# else
-# define CHECK_GCC_ASM__
-# endif
-#elif defined(ETHR_HAVE_LIBATOMIC_OPS)
-# define CHECK_LIBATOMIC_OPS__
+AC_ARG_WITH(with_sparc_memory_order,
+ AS_HELP_STRING([--with-sparc-memory-order=TSO|PSO|RMO],
+ [specify sparc memory order (defaults to RMO)]))
+
+ETHR_X86_SSE2_ASM=no
+case "$GCC-$ac_cv_sizeof_void_p-$host_cpu" in
+ yes-4-i86pc | yes-4-i*86 | yes-4-x86_64 | yes-4-amd64)
+ AC_MSG_CHECKING([for gcc sse2 asm support])
+ save_CFLAGS="$CFLAGS"
+ CFLAGS="$CFLAGS -msse2"
+ gcc_sse2_asm=no
+ AC_TRY_COMPILE([],
+ [
+ long long x, *y;
+ __asm__ __volatile__("movq %1, %0\n\t" : "=x"(x) : "m"(*y) : "memory");
+ ],
+ [gcc_sse2_asm=yes])
+ CFLAGS="$save_CFLAGS"
+ AC_MSG_RESULT([$gcc_sse2_asm])
+ if test "$gcc_sse2_asm" = "yes"; then
+ AC_DEFINE(ETHR_GCC_HAVE_SSE2_ASM_SUPPORT, 1, [Define if you use a gcc that supports -msse2 and understand sse2 specific asm statements])
+ ETHR_X86_SSE2_ASM=yes
+ fi
+ ;;
+ *)
+ ;;
+esac
+
+case "$GCC-$host_cpu" in
+ yes-i86pc | yes-i*86 | yes-x86_64 | yes-amd64)
+ gcc_dw_cmpxchg_asm=no
+ AC_MSG_CHECKING([for gcc double word cmpxchg asm support])
+ AC_TRY_COMPILE([],
+ [
+ char xchgd;
+ long new[2], xchg[2], *p;
+ __asm__ __volatile__(
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ "pushl %%ebx\n\t"
+ "movl %8, %%ebx\n\t"
#endif
-#if defined(CHECK_LIBATOMIC_OPS__)
-#include "atomic_ops.h"
+#if ETHR_SIZEOF_PTR == 4
+ "lock; cmpxchg8b %0\n\t"
+#else
+ "lock; cmpxchg16b %0\n\t"
#endif
-int main(void)
-{
-#if defined(CHECK_GCC_ASM__)
- __asm__ __volatile__("mfence" : : : "memory");
-#elif defined(CHECK_LIBATOMIC_OPS__)
- AO_nop_full();
+ "setz %3\n\t"
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ "popl %%ebx\n\t"
#endif
- return 0;
-}
+ : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+ : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new[1]),
+#if ETHR_SIZEOF_PTR == 4 && defined(__PIC__) && __PIC__
+ "r"(new[0])
+#else
+ "b"(new[0])
+#endif
+ : "cc", "memory");
+
],
- [enable_ethread_pre_pentium4_compatibility=no],
- [enable_ethread_pre_pentium4_compatibility=yes],
- [enable_ethread_pre_pentium4_compatibility=no])
- AC_MSG_RESULT([$enable_ethread_pre_pentium4_compatibility]);;
+ [gcc_dw_cmpxchg_asm=yes])
+ if test $gcc_dw_cmpxchg_asm = no && test $ac_cv_sizeof_void_p = 4; then
+ AC_TRY_COMPILE([],
+ [
+ char xchgd;
+ long new[2], xchg[2], *p;
+#if !defined(__PIC__) || !__PIC__
+# error nope
+#endif
+ __asm__ __volatile__(
+ "pushl %%ebx\n\t"
+ "movl (%7), %%ebx\n\t"
+ "movl 4(%7), %%ecx\n\t"
+ "lock; cmpxchg8b %0\n\t"
+ "setz %3\n\t"
+ "popl %%ebx\n\t"
+ : "=m"(*p), "=d"(xchg[1]), "=a"(xchg[0]), "=c"(xchgd)
+ : "m"(*p), "1"(xchg[1]), "2"(xchg[0]), "3"(new)
+ : "cc", "memory");
+
+ ],
+ [gcc_dw_cmpxchg_asm=yes])
+ if test "$gcc_dw_cmpxchg_asm" = "yes"; then
+ AC_DEFINE(ETHR_CMPXCHG8B_REGISTER_SHORTAGE, 1, [Define if you get a register shortage with cmpxchg8b and position independent code])
+ fi
+ fi
+ AC_MSG_RESULT([$gcc_dw_cmpxchg_asm])
+ if test "$gcc_dw_cmpxchg_asm" = "yes"; then
+ AC_DEFINE(ETHR_GCC_HAVE_DW_CMPXCHG_ASM_SUPPORT, 1, [Define if you use a gcc that supports the double word cmpxchg instruction])
+ fi;;
*)
;;
esac
-test $enable_ethread_pre_pentium4_compatibility = yes &&
- AC_DEFINE(ETHR_PRE_PENTIUM4_COMPAT, 1, [Define if you want compatibilty with x86 processors before pentium4.])
+
AC_DEFINE(ETHR_HAVE_ETHREAD_DEFINES, 1, \
[Define if you have all ethread defines])
@@ -1309,6 +1374,7 @@ AC_SUBST(ETHR_LIB_NAME)
AC_SUBST(ETHR_DEFS)
AC_SUBST(ETHR_THR_LIB_BASE)
AC_SUBST(ETHR_THR_LIB_BASE_DIR)
+AC_SUBST(ETHR_X86_SSE2_ASM)
])
View
64 erts/emulator/beam/erl_bif_info.c
@@ -2545,6 +2545,70 @@ BIF_RETTYPE system_info_1(BIF_ALIST_1)
hp = hsz ? HAlloc(BIF_P, hsz) : NULL;
res = erts_bld_uint(&hp, NULL, erts_dist_buf_busy_limit);
BIF_RET(res);
+ } else if (ERTS_IS_ATOM_STR("print_ethread_info", BIF_ARG_1)) {
+ int i;
+ char **str;
+#ifdef ETHR_NATIVE_ATOMIC32_IMPL
+ erts_printf("32-bit native atomics: %s\n",
+ ETHR_NATIVE_ATOMIC32_IMPL);
+ str = ethr_native_atomic32_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_atomic32_%s()\n", str[i]);
+#endif
+#ifdef ETHR_NATIVE_ATOMIC64_IMPL
+ erts_printf("64-bit native atomics: %s\n",
+ ETHR_NATIVE_ATOMIC64_IMPL);
+ str = ethr_native_atomic64_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_atomic64_%s()\n", str[i]);
+#endif
+#ifdef ETHR_NATIVE_DW_ATOMIC_IMPL
+ if (ethr_have_native_dw_atomic()) {
+ erts_printf("Double word native atomics: %s\n",
+ ETHR_NATIVE_DW_ATOMIC_IMPL);
+ str = ethr_native_dw_atomic_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_dw_atomic_%s()\n", str[i]);
+ str = ethr_native_su_dw_atomic_ops();
+ for (i = 0; str[i]; i++)
+ erts_printf("ethr_native_su_dw_atomic_%s()\n", str[i]);
+ }
+#endif
+#ifdef ETHR_NATIVE_SPINLOCK_IMPL
+ erts_printf("Native spin-locks: %s\n", ETHR_NATIVE_SPINLOCK_IMPL);
+#endif
+#ifdef ETHR_NATIVE_RWSPINLOCK_IMPL
+ erts_printf("Native rwspin-locks: %s\n", ETHR_NATIVE_RWSPINLOCK_IMPL);
+#endif
+#ifdef ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+ erts_printf("SSE2 support: %s\n", (ETHR_X86_RUNTIME_CONF_HAVE_SSE2__
+ ? "yes" : "no"));
+#endif
+#ifdef ETHR_X86_OUT_OF_ORDER
+ erts_printf("x86"
+#ifdef ARCH_64
+ "_64"
+#endif
+ " out of order\n");
+#endif
+#ifdef ETHR_SPARC_TSO
+ erts_printf("Sparc TSO\n");
+#endif
+#ifdef ETHR_SPARC_PSO
+ erts_printf("Sparc PSO\n");
+#endif
+#ifdef ETHR_SPARC_RMO
+ erts_printf("Sparc RMO\n");
+#endif
+#if defined(ETHR_PPC_HAVE_LWSYNC)
+ erts_printf("Have lwsync instruction: yes\n");
+#elif defined(ETHR_PPC_HAVE_NO_LWSYNC)
+ erts_printf("Have lwsync instruction: no\n");
+#elif defined(ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__)
+ erts_printf("Have lwsync instruction: %s (runtime test)\n",
+ ETHR_PPC_RUNTIME_CONF_HAVE_LWSYNC__ ? "yes" : "no");
+#endif
+ BIF_RET(am_true);
}
BIF_ERROR(BIF_P, BADARG);
View
6 erts/emulator/beam/erl_process_lock.h
@@ -1,7 +1,7 @@
/*
* %CopyrightBegin%
*
- * Copyright Ericsson AB 2007-2010. All Rights Reserved.
+ * Copyright Ericsson AB 2007-2011. All Rights Reserved.
*
* The contents of this file are subject to the Erlang Public License,
* Version 1.1, (the "License"); you may not use this file except in
@@ -41,10 +41,10 @@
#define ERTS_PROC_LOCK_SPINLOCK_IMPL 0
#define ERTS_PROC_LOCK_MUTEX_IMPL 0
-#if defined(ETHR_HAVE_OPTIMIZED_ATOMIC_OPS)
+#if defined(ETHR_HAVE_32BIT_NATIVE_ATOMIC_OPS)
# undef ERTS_PROC_LOCK_ATOMIC_IMPL
# define ERTS_PROC_LOCK_ATOMIC_IMPL 1
-#elif defined(ETHR_HAVE_OPTIMIZED_SPINLOCK)
+#elif defined(ETHR_HAVE_NATIVE_SPINLOCKS)
# undef ERTS_PROC_LOCK_SPINLOCK_IMPL
# define ERTS_PROC_LOCK_SPINLOCK_IMPL 1
#else
View
1,006 erts/emulator/beam/erl_smp.h
@@ -54,8 +54,9 @@ typedef erts_cnd_t erts_smp_cnd_t;
typedef erts_rwmtx_opt_t erts_smp_rwmtx_opt_t;
typedef erts_rwmtx_t erts_smp_rwmtx_t;
typedef erts_tsd_key_t erts_smp_tsd_key_t;
-typedef erts_atomic_t erts_smp_atomic_t;
-typedef erts_atomic32_t erts_smp_atomic32_t;
+#define erts_smp_dw_atomic_t erts_dw_atomic_t
+#define erts_smp_atomic_t erts_atomic_t
+#define erts_smp_atomic32_t erts_atomic32_t
typedef erts_spinlock_t erts_smp_spinlock_t;
typedef erts_rwlock_t erts_smp_rwlock_t;
void erts_thr_fatal_error(int, char *); /* implemented in erl_init.c */
@@ -83,8 +84,9 @@ typedef struct {
} erts_smp_rwmtx_opt_t;
typedef int erts_smp_rwmtx_t;
typedef int erts_smp_tsd_key_t;
-typedef SWord erts_smp_atomic_t;
-typedef Uint32 erts_smp_atomic32_t;
+#define erts_smp_dw_atomic_t erts_no_dw_atomic_t
+#define erts_smp_atomic_t erts_no_atomic_t
+#define erts_smp_atomic32_t erts_no_atomic32_t
#if __GNUC__ > 2
typedef struct { } erts_smp_spinlock_t;
typedef struct { } erts_smp_rwlock_t;
@@ -160,82 +162,6 @@ ERTS_GLB_INLINE int erts_smp_rwmtx_tryrwlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_smp_rwmtx_rwunlock(erts_smp_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rlocked(erts_smp_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_smp_atomic_init(erts_smp_atomic_t *var,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read(erts_smp_atomic_t *var);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_inctest(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE void erts_smp_atomic_inc(erts_smp_atomic_t *incp);
-ERTS_GLB_INLINE void erts_smp_atomic_dec(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_addtest(erts_smp_atomic_t *addp,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_smp_atomic_add(erts_smp_atomic_t *addp,
- erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp,
- erts_aint_t new);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t expected);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_bor(erts_smp_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_band(erts_smp_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_read_acqb(erts_smp_atomic_t *var);
-ERTS_GLB_INLINE void erts_smp_atomic_set_relb(erts_smp_atomic_t *var,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE erts_aint_t erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read(erts_smp_atomic32_t *var);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_inc(erts_smp_atomic32_t *incp);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
ERTS_GLB_INLINE void erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock,
char *name,
Eterm extra);
@@ -279,6 +205,464 @@ ERTS_GLB_INLINE void erts_smp_thr_sigmask(int how,
ERTS_GLB_INLINE void erts_smp_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef ERTS_THR_HAVE_SIG_FUNCS */
+/*
+ * Functions implementing atomic operations with with no (nob),
+ * full (mb), acquire (acqb), release (relb), read (rb), and
+ * write (wb) memory barriers.
+ *
+ * If SMP support has been disabled, they are mapped to functions
+ * that performs the same operation, but aren't atomic and don't
+ * imply memory barriers.
+ */
+
+#ifdef ERTS_SMP
+
+/* Double word size atomics */
+
+#define erts_smp_dw_atomic_init_nob erts_dw_atomic_init_nob
+#define erts_smp_dw_atomic_set_nob erts_dw_atomic_set_nob
+#define erts_smp_dw_atomic_read_nob erts_dw_atomic_read_nob
+#define erts_smp_dw_atomic_cmpxchg_nob erts_dw_atomic_cmpxchg_nob
+
+#define erts_smp_dw_atomic_init_mb erts_dw_atomic_init_mb
+#define erts_smp_dw_atomic_set_mb erts_dw_atomic_set_mb
+#define erts_smp_dw_atomic_read_mb erts_dw_atomic_read_mb
+#define erts_smp_dw_atomic_cmpxchg_mb erts_dw_atomic_cmpxchg_mb
+
+#define erts_smp_dw_atomic_init_acqb erts_dw_atomic_init_acqb
+#define erts_smp_dw_atomic_set_acqb erts_dw_atomic_set_acqb
+#define erts_smp_dw_atomic_read_acqb erts_dw_atomic_read_acqb
+#define erts_smp_dw_atomic_cmpxchg_acqb erts_dw_atomic_cmpxchg_acqb
+
+#define erts_smp_dw_atomic_init_relb erts_dw_atomic_init_relb
+#define erts_smp_dw_atomic_set_relb erts_dw_atomic_set_relb
+#define erts_smp_dw_atomic_read_relb erts_dw_atomic_read_relb
+#define erts_smp_dw_atomic_cmpxchg_relb erts_dw_atomic_cmpxchg_relb
+
+#define erts_smp_dw_atomic_init_rb erts_dw_atomic_init_rb
+#define erts_smp_dw_atomic_set_rb erts_dw_atomic_set_rb
+#define erts_smp_dw_atomic_read_rb erts_dw_atomic_read_rb
+#define erts_smp_dw_atomic_cmpxchg_rb erts_dw_atomic_cmpxchg_rb
+
+#define erts_smp_dw_atomic_init_wb erts_dw_atomic_init_wb
+#define erts_smp_dw_atomic_set_wb erts_dw_atomic_set_wb
+#define erts_smp_dw_atomic_read_wb erts_dw_atomic_read_wb
+#define erts_smp_dw_atomic_cmpxchg_wb erts_dw_atomic_cmpxchg_wb
+
+/* Word size atomics */
+
+#define erts_smp_atomic_init_nob erts_atomic_init_nob
+#define erts_smp_atomic_set_nob erts_atomic_set_nob
+#define erts_smp_atomic_read_nob erts_atomic_read_nob
+#define erts_smp_atomic_inc_read_nob erts_atomic_inc_read_nob
+#define erts_smp_atomic_dec_read_nob erts_atomic_dec_read_nob
+#define erts_smp_atomic_inc_nob erts_atomic_inc_nob
+#define erts_smp_atomic_dec_nob erts_atomic_dec_nob
+#define erts_smp_atomic_add_read_nob erts_atomic_add_read_nob
+#define erts_smp_atomic_add_nob erts_atomic_add_nob
+#define erts_smp_atomic_read_bor_nob erts_atomic_read_bor_nob
+#define erts_smp_atomic_read_band_nob erts_atomic_read_band_nob
+#define erts_smp_atomic_xchg_nob erts_atomic_xchg_nob
+#define erts_smp_atomic_cmpxchg_nob erts_atomic_cmpxchg_nob
+
+#define erts_smp_atomic_init_mb erts_atomic_init_mb
+#define erts_smp_atomic_set_mb erts_atomic_set_mb
+#define erts_smp_atomic_read_mb erts_atomic_read_mb
+#define erts_smp_atomic_inc_read_mb erts_atomic_inc_read_mb
+#define erts_smp_atomic_dec_read_mb erts_atomic_dec_read_mb
+#define erts_smp_atomic_inc_mb erts_atomic_inc_mb
+#define erts_smp_atomic_dec_mb erts_atomic_dec_mb
+#define erts_smp_atomic_add_read_mb erts_atomic_add_read_mb
+#define erts_smp_atomic_add_mb erts_atomic_add_mb
+#define erts_smp_atomic_read_bor_mb erts_atomic_read_bor_mb
+#define erts_smp_atomic_read_band_mb erts_atomic_read_band_mb
+#define erts_smp_atomic_xchg_mb erts_atomic_xchg_mb
+#define erts_smp_atomic_cmpxchg_mb erts_atomic_cmpxchg_mb
+
+#define erts_smp_atomic_init_acqb erts_atomic_init_acqb
+#define erts_smp_atomic_set_acqb erts_atomic_set_acqb
+#define erts_smp_atomic_read_acqb erts_atomic_read_acqb
+#define erts_smp_atomic_inc_read_acqb erts_atomic_inc_read_acqb
+#define erts_smp_atomic_dec_read_acqb erts_atomic_dec_read_acqb
+#define erts_smp_atomic_inc_acqb erts_atomic_inc_acqb
+#define erts_smp_atomic_dec_acqb erts_atomic_dec_acqb
+#define erts_smp_atomic_add_read_acqb erts_atomic_add_read_acqb
+#define erts_smp_atomic_add_acqb erts_atomic_add_acqb
+#define erts_smp_atomic_read_bor_acqb erts_atomic_read_bor_acqb
+#define erts_smp_atomic_read_band_acqb erts_atomic_read_band_acqb
+#define erts_smp_atomic_xchg_acqb erts_atomic_xchg_acqb
+#define erts_smp_atomic_cmpxchg_acqb erts_atomic_cmpxchg_acqb
+
+#define erts_smp_atomic_init_relb erts_atomic_init_relb
+#define erts_smp_atomic_set_relb erts_atomic_set_relb
+#define erts_smp_atomic_read_relb erts_atomic_read_relb
+#define erts_smp_atomic_inc_read_relb erts_atomic_inc_read_relb
+#define erts_smp_atomic_dec_read_relb erts_atomic_dec_read_relb
+#define erts_smp_atomic_inc_relb erts_atomic_inc_relb
+#define erts_smp_atomic_dec_relb erts_atomic_dec_relb
+#define erts_smp_atomic_add_read_relb erts_atomic_add_read_relb
+#define erts_smp_atomic_add_relb erts_atomic_add_relb
+#define erts_smp_atomic_read_bor_relb erts_atomic_read_bor_relb
+#define erts_smp_atomic_read_band_relb erts_atomic_read_band_relb
+#define erts_smp_atomic_xchg_relb erts_atomic_xchg_relb
+#define erts_smp_atomic_cmpxchg_relb erts_atomic_cmpxchg_relb
+
+#define erts_smp_atomic_init_rb erts_atomic_init_rb
+#define erts_smp_atomic_set_rb erts_atomic_set_rb
+#define erts_smp_atomic_read_rb erts_atomic_read_rb
+#define erts_smp_atomic_inc_read_rb erts_atomic_inc_read_rb
+#define erts_smp_atomic_dec_read_rb erts_atomic_dec_read_rb
+#define erts_smp_atomic_inc_rb erts_atomic_inc_rb
+#define erts_smp_atomic_dec_rb erts_atomic_dec_rb
+#define erts_smp_atomic_add_read_rb erts_atomic_add_read_rb
+#define erts_smp_atomic_add_rb erts_atomic_add_rb
+#define erts_smp_atomic_read_bor_rb erts_atomic_read_bor_rb
+#define erts_smp_atomic_read_band_rb erts_atomic_read_band_rb
+#define erts_smp_atomic_xchg_rb erts_atomic_xchg_rb
+#define erts_smp_atomic_cmpxchg_rb erts_atomic_cmpxchg_rb
+
+#define erts_smp_atomic_init_wb erts_atomic_init_wb
+#define erts_smp_atomic_set_wb erts_atomic_set_wb
+#define erts_smp_atomic_read_wb erts_atomic_read_wb
+#define erts_smp_atomic_inc_read_wb erts_atomic_inc_read_wb
+#define erts_smp_atomic_dec_read_wb erts_atomic_dec_read_wb
+#define erts_smp_atomic_inc_wb erts_atomic_inc_wb
+#define erts_smp_atomic_dec_wb erts_atomic_dec_wb
+#define erts_smp_atomic_add_read_wb erts_atomic_add_read_wb
+#define erts_smp_atomic_add_wb erts_atomic_add_wb
+#define erts_smp_atomic_read_bor_wb erts_atomic_read_bor_wb
+#define erts_smp_atomic_read_band_wb erts_atomic_read_band_wb
+#define erts_smp_atomic_xchg_wb erts_atomic_xchg_wb
+#define erts_smp_atomic_cmpxchg_wb erts_atomic_cmpxchg_wb
+
+/* 32-bit atomics */
+
+#define erts_smp_atomic32_init_nob erts_atomic32_init_nob
+#define erts_smp_atomic32_set_nob erts_atomic32_set_nob
+#define erts_smp_atomic32_read_nob erts_atomic32_read_nob
+#define erts_smp_atomic32_inc_read_nob erts_atomic32_inc_read_nob
+#define erts_smp_atomic32_dec_read_nob erts_atomic32_dec_read_nob
+#define erts_smp_atomic32_inc_nob erts_atomic32_inc_nob
+#define erts_smp_atomic32_dec_nob erts_atomic32_dec_nob
+#define erts_smp_atomic32_add_read_nob erts_atomic32_add_read_nob
+#define erts_smp_atomic32_add_nob erts_atomic32_add_nob
+#define erts_smp_atomic32_read_bor_nob erts_atomic32_read_bor_nob
+#define erts_smp_atomic32_read_band_nob erts_atomic32_read_band_nob
+#define erts_smp_atomic32_xchg_nob erts_atomic32_xchg_nob
+#define erts_smp_atomic32_cmpxchg_nob erts_atomic32_cmpxchg_nob
+
+#define erts_smp_atomic32_init_mb erts_atomic32_init_mb
+#define erts_smp_atomic32_set_mb erts_atomic32_set_mb
+#define erts_smp_atomic32_read_mb erts_atomic32_read_mb
+#define erts_smp_atomic32_inc_read_mb erts_atomic32_inc_read_mb
+#define erts_smp_atomic32_dec_read_mb erts_atomic32_dec_read_mb
+#define erts_smp_atomic32_inc_mb erts_atomic32_inc_mb
+#define erts_smp_atomic32_dec_mb erts_atomic32_dec_mb
+#define erts_smp_atomic32_add_read_mb erts_atomic32_add_read_mb
+#define erts_smp_atomic32_add_mb erts_atomic32_add_mb
+#define erts_smp_atomic32_read_bor_mb erts_atomic32_read_bor_mb
+#define erts_smp_atomic32_read_band_mb erts_atomic32_read_band_mb
+#define erts_smp_atomic32_xchg_mb erts_atomic32_xchg_mb
+#define erts_smp_atomic32_cmpxchg_mb erts_atomic32_cmpxchg_mb
+
+#define erts_smp_atomic32_init_acqb erts_atomic32_init_acqb
+#define erts_smp_atomic32_set_acqb erts_atomic32_set_acqb
+#define erts_smp_atomic32_read_acqb erts_atomic32_read_acqb
+#define erts_smp_atomic32_inc_read_acqb erts_atomic32_inc_read_acqb
+#define erts_smp_atomic32_dec_read_acqb erts_atomic32_dec_read_acqb
+#define erts_smp_atomic32_inc_acqb erts_atomic32_inc_acqb
+#define erts_smp_atomic32_dec_acqb erts_atomic32_dec_acqb
+#define erts_smp_atomic32_add_read_acqb erts_atomic32_add_read_acqb
+#define erts_smp_atomic32_add_acqb erts_atomic32_add_acqb
+#define erts_smp_atomic32_read_bor_acqb erts_atomic32_read_bor_acqb
+#define erts_smp_atomic32_read_band_acqb erts_atomic32_read_band_acqb
+#define erts_smp_atomic32_xchg_acqb erts_atomic32_xchg_acqb
+#define erts_smp_atomic32_cmpxchg_acqb erts_atomic32_cmpxchg_acqb
+
+#define erts_smp_atomic32_init_relb erts_atomic32_init_relb
+#define erts_smp_atomic32_set_relb erts_atomic32_set_relb
+#define erts_smp_atomic32_read_relb erts_atomic32_read_relb
+#define erts_smp_atomic32_inc_read_relb erts_atomic32_inc_read_relb
+#define erts_smp_atomic32_dec_read_relb erts_atomic32_dec_read_relb
+#define erts_smp_atomic32_inc_relb erts_atomic32_inc_relb
+#define erts_smp_atomic32_dec_relb erts_atomic32_dec_relb
+#define erts_smp_atomic32_add_read_relb erts_atomic32_add_read_relb
+#define erts_smp_atomic32_add_relb erts_atomic32_add_relb
+#define erts_smp_atomic32_read_bor_relb erts_atomic32_read_bor_relb
+#define erts_smp_atomic32_read_band_relb erts_atomic32_read_band_relb
+#define erts_smp_atomic32_xchg_relb erts_atomic32_xchg_relb
+#define erts_smp_atomic32_cmpxchg_relb erts_atomic32_cmpxchg_relb
+
+#define erts_smp_atomic32_init_rb erts_atomic32_init_rb
+#define erts_smp_atomic32_set_rb erts_atomic32_set_rb
+#define erts_smp_atomic32_read_rb erts_atomic32_read_rb
+#define erts_smp_atomic32_inc_read_rb erts_atomic32_inc_read_rb
+#define erts_smp_atomic32_dec_read_rb erts_atomic32_dec_read_rb
+#define erts_smp_atomic32_inc_rb erts_atomic32_inc_rb
+#define erts_smp_atomic32_dec_rb erts_atomic32_dec_rb
+#define erts_smp_atomic32_add_read_rb erts_atomic32_add_read_rb
+#define erts_smp_atomic32_add_rb erts_atomic32_add_rb
+#define erts_smp_atomic32_read_bor_rb erts_atomic32_read_bor_rb
+#define erts_smp_atomic32_read_band_rb erts_atomic32_read_band_rb
+#define erts_smp_atomic32_xchg_rb erts_atomic32_xchg_rb
+#define erts_smp_atomic32_cmpxchg_rb erts_atomic32_cmpxchg_rb
+
+#define erts_smp_atomic32_init_wb erts_atomic32_init_wb
+#define erts_smp_atomic32_set_wb erts_atomic32_set_wb
+#define erts_smp_atomic32_read_wb erts_atomic32_read_wb
+#define erts_smp_atomic32_inc_read_wb erts_atomic32_inc_read_wb
+#define erts_smp_atomic32_dec_read_wb erts_atomic32_dec_read_wb
+#define erts_smp_atomic32_inc_wb erts_atomic32_inc_wb
+#define erts_smp_atomic32_dec_wb erts_atomic32_dec_wb
+#define erts_smp_atomic32_add_read_wb erts_atomic32_add_read_wb
+#define erts_smp_atomic32_add_wb erts_atomic32_add_wb
+#define erts_smp_atomic32_read_bor_wb erts_atomic32_read_bor_wb
+#define erts_smp_atomic32_read_band_wb erts_atomic32_read_band_wb
+#define erts_smp_atomic32_xchg_wb erts_atomic32_xchg_wb
+#define erts_smp_atomic32_cmpxchg_wb erts_atomic32_cmpxchg_wb
+
+#else /* !ERTS_SMP */
+
+/* Double word size atomics */
+
+#define erts_smp_dw_atomic_init_nob erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_set_nob erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_nob erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_mb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_mb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_mb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_acqb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_acqb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_acqb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_relb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_relb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_relb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_rb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_rb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_rb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
+
+#define erts_smp_dw_atomic_init_wb erts_no_dw_atomic_init
+#define erts_smp_dw_atomic_set_wb erts_no_dw_atomic_set
+#define erts_smp_dw_atomic_read_wb erts_no_dw_atomic_read
+#define erts_smp_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
+
+/* Word size atomics */
+
+#define erts_smp_atomic_init_nob erts_no_atomic_set
+#define erts_smp_atomic_set_nob erts_no_atomic_set
+#define erts_smp_atomic_read_nob erts_no_atomic_read
+#define erts_smp_atomic_inc_read_nob erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_nob erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_nob erts_no_atomic_inc
+#define erts_smp_atomic_dec_nob erts_no_atomic_dec
+#define erts_smp_atomic_add_read_nob erts_no_atomic_add_read
+#define erts_smp_atomic_add_nob erts_no_atomic_add
+#define erts_smp_atomic_read_bor_nob erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_nob erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_nob erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_mb erts_no_atomic_set
+#define erts_smp_atomic_set_mb erts_no_atomic_set
+#define erts_smp_atomic_read_mb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_mb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_mb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_mb erts_no_atomic_inc
+#define erts_smp_atomic_dec_mb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_mb erts_no_atomic_add_read
+#define erts_smp_atomic_add_mb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_mb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_mb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_mb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_acqb erts_no_atomic_set
+#define erts_smp_atomic_set_acqb erts_no_atomic_set
+#define erts_smp_atomic_read_acqb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_acqb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_acqb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_acqb erts_no_atomic_inc
+#define erts_smp_atomic_dec_acqb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_acqb erts_no_atomic_add_read
+#define erts_smp_atomic_add_acqb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_acqb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_acqb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_acqb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_relb erts_no_atomic_set
+#define erts_smp_atomic_set_relb erts_no_atomic_set
+#define erts_smp_atomic_read_relb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_relb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_relb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_relb erts_no_atomic_inc
+#define erts_smp_atomic_dec_relb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_relb erts_no_atomic_add_read
+#define erts_smp_atomic_add_relb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_relb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_relb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_relb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_rb erts_no_atomic_set
+#define erts_smp_atomic_set_rb erts_no_atomic_set
+#define erts_smp_atomic_read_rb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_rb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_rb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_rb erts_no_atomic_inc
+#define erts_smp_atomic_dec_rb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_rb erts_no_atomic_add_read
+#define erts_smp_atomic_add_rb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_rb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_rb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_rb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+
+#define erts_smp_atomic_init_wb erts_no_atomic_set
+#define erts_smp_atomic_set_wb erts_no_atomic_set
+#define erts_smp_atomic_read_wb erts_no_atomic_read
+#define erts_smp_atomic_inc_read_wb erts_no_atomic_inc_read
+#define erts_smp_atomic_dec_read_wb erts_no_atomic_dec_read
+#define erts_smp_atomic_inc_wb erts_no_atomic_inc
+#define erts_smp_atomic_dec_wb erts_no_atomic_dec
+#define erts_smp_atomic_add_read_wb erts_no_atomic_add_read
+#define erts_smp_atomic_add_wb erts_no_atomic_add
+#define erts_smp_atomic_read_bor_wb erts_no_atomic_read_bor
+#define erts_smp_atomic_read_band_wb erts_no_atomic_read_band
+#define erts_smp_atomic_xchg_wb erts_no_atomic_xchg
+#define erts_smp_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+
+/* 32-bit atomics */
+
+#define erts_smp_atomic32_init_nob erts_no_atomic32_set
+#define erts_smp_atomic32_set_nob erts_no_atomic32_set
+#define erts_smp_atomic32_read_nob erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_nob erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_nob erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_nob erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_nob erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_nob erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_nob erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_nob erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_nob erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_nob erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_mb erts_no_atomic32_set
+#define erts_smp_atomic32_set_mb erts_no_atomic32_set
+#define erts_smp_atomic32_read_mb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_mb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_mb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_mb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_mb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_mb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_mb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_mb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_mb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_mb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_acqb erts_no_atomic32_set
+#define erts_smp_atomic32_set_acqb erts_no_atomic32_set
+#define erts_smp_atomic32_read_acqb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_acqb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_acqb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_acqb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_acqb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_acqb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_acqb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_acqb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_acqb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_acqb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_relb erts_no_atomic32_set
+#define erts_smp_atomic32_set_relb erts_no_atomic32_set
+#define erts_smp_atomic32_read_relb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_relb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_relb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_relb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_relb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_relb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_relb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_relb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_relb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_relb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_rb erts_no_atomic32_set
+#define erts_smp_atomic32_set_rb erts_no_atomic32_set
+#define erts_smp_atomic32_read_rb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_rb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_rb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_rb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_rb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_rb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_rb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_rb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_rb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_rb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+
+#define erts_smp_atomic32_init_wb erts_no_atomic32_set
+#define erts_smp_atomic32_set_wb erts_no_atomic32_set
+#define erts_smp_atomic32_read_wb erts_no_atomic32_read
+#define erts_smp_atomic32_inc_read_wb erts_no_atomic32_inc_read
+#define erts_smp_atomic32_dec_read_wb erts_no_atomic32_dec_read
+#define erts_smp_atomic32_inc_wb erts_no_atomic32_inc
+#define erts_smp_atomic32_dec_wb erts_no_atomic32_dec
+#define erts_smp_atomic32_add_read_wb erts_no_atomic32_add_read
+#define erts_smp_atomic32_add_wb erts_no_atomic32_add
+#define erts_smp_atomic32_read_bor_wb erts_no_atomic32_read_bor
+#define erts_smp_atomic32_read_band_wb erts_no_atomic32_read_band
+#define erts_smp_atomic32_xchg_wb erts_no_atomic32_xchg
+#define erts_smp_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+
+#endif /* !ERTS_SMP */
+
+#ifndef ERTS_NO_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#define erts_smp_atomic_init erts_smp_atomic_init_nob
+#define erts_smp_atomic_set erts_smp_atomic_set_nob
+#define erts_smp_atomic_read erts_smp_atomic_read_nob
+#define erts_smp_atomic_inctest erts_smp_atomic_inc_read_mb
+#define erts_smp_atomic_dectest erts_smp_atomic_dec_read_mb
+#define erts_smp_atomic_inc erts_smp_atomic_inc_mb
+#define erts_smp_atomic_dec erts_smp_atomic_dec_mb
+#define erts_smp_atomic_addtest erts_smp_atomic_add_read_mb
+#define erts_smp_atomic_add erts_smp_atomic_add_mb
+#define erts_smp_atomic_xchg erts_smp_atomic_xchg_mb
+#define erts_smp_atomic_cmpxchg erts_smp_atomic_cmpxchg_mb
+#define erts_smp_atomic_bor erts_smp_atomic_read_bor_mb
+#define erts_smp_atomic_band erts_smp_atomic_read_band_mb
+
+#define erts_smp_atomic32_init erts_smp_atomic32_init_nob
+#define erts_smp_atomic32_set erts_smp_atomic32_set_nob
+#define erts_smp_atomic32_read erts_smp_atomic32_read_nob
+#define erts_smp_atomic32_inctest erts_smp_atomic32_inc_read_mb
+#define erts_smp_atomic32_dectest erts_smp_atomic32_dec_read_mb
+#define erts_smp_atomic32_inc erts_smp_atomic32_inc_mb
+#define erts_smp_atomic32_dec erts_smp_atomic32_dec_mb
+#define erts_smp_atomic32_addtest erts_smp_atomic32_add_read_mb
+#define erts_smp_atomic32_add erts_smp_atomic32_add_mb
+#define erts_smp_atomic32_xchg erts_smp_atomic32_xchg_mb
+#define erts_smp_atomic32_cmpxchg erts_smp_atomic32_cmpxchg_mb
+#define erts_smp_atomic32_bor erts_smp_atomic32_read_bor_mb
+#define erts_smp_atomic32_band erts_smp_atomic32_read_band_mb
+
+#endif
+
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
@@ -655,434 +1039,6 @@ erts_smp_lc_rwmtx_is_rwlocked(erts_smp_rwmtx_t *mtx)
}
ERTS_GLB_INLINE void
-erts_smp_atomic_init(erts_smp_atomic_t *var, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_init(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_set(erts_smp_atomic_t *var, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_set(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_read(erts_smp_atomic_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic_read(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_inctest(erts_smp_atomic_t *incp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_inctest(incp);
-#else
- return ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_dectest(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_dectest(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_inc(erts_smp_atomic_t *incp)
-{
-#ifdef ERTS_SMP
- erts_atomic_inc(incp);
-#else
- ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_dec(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic_dec(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_addtest(erts_smp_atomic_t *addp, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- return erts_atomic_addtest(addp, i);
-#else
- return *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_add(erts_smp_atomic_t *addp, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_add(addp, i);
-#else
- *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_xchg(erts_smp_atomic_t *xchgp, erts_aint_t new)
-{
-#ifdef ERTS_SMP
- return erts_atomic_xchg(xchgp, new);
-#else
- erts_aint_t old;
- old = *xchgp;
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_cmpxchg(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t expected)
-{
-#ifdef ERTS_SMP
- return erts_atomic_cmpxchg(xchgp, new, expected);
-#else
- erts_aint_t old = *xchgp;
- if (old == expected)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_bor(erts_smp_atomic_t *var, erts_aint_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic_bor(var, mask);
-#else
- erts_aint_t old;
- old = *var;
- *var |= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_band(erts_smp_atomic_t *var, erts_aint_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic_band(var, mask);
-#else
- erts_aint_t old;
- old = *var;
- *var &= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_read_acqb(erts_smp_atomic_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic_read_acqb(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_set_relb(erts_smp_atomic_t *var, erts_aint_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic_set_relb(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic_dec_relb(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic_dec_relb(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_dectest_relb(erts_smp_atomic_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_dectest_relb(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_cmpxchg_acqb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_cmpxchg_acqb(xchgp, new, exp);
-#else
- erts_aint_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint_t
-erts_smp_atomic_cmpxchg_relb(erts_smp_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic_cmpxchg_relb(xchgp, new, exp);
-#else
- erts_aint_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_init(erts_smp_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_init(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set(erts_smp_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_set(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read(erts_smp_atomic32_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_read(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_inctest(erts_smp_atomic32_t *incp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_inctest(incp);
-#else
- return ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_dectest(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_inc(erts_smp_atomic32_t *incp)
-{
-#ifdef ERTS_SMP
- erts_atomic32_inc(incp);
-#else
- ++(*incp);
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic32_dec(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_addtest(erts_smp_atomic32_t *addp, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_addtest(addp, i);
-#else
- return *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_add(erts_smp_atomic32_t *addp, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_add(addp, i);
-#else
- *addp += i;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_xchg(erts_smp_atomic32_t *xchgp, erts_aint32_t new)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_xchg(xchgp, new);
-#else
- erts_aint32_t old;
- old = *xchgp;
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_cmpxchg(xchgp, new, expected);
-#else
- erts_aint32_t old = *xchgp;
- if (old == expected)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_bor(erts_smp_atomic32_t *var, erts_aint32_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_bor(var, mask);
-#else
- erts_aint32_t old;
- old = *var;
- *var |= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_band(erts_smp_atomic32_t *var, erts_aint32_t mask)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_band(var, mask);
-#else
- erts_aint32_t old;
- old = *var;
- *var &= mask;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_read_acqb(erts_smp_atomic32_t *var)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_read_acqb(var);
-#else
- return *var;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_set_relb(erts_smp_atomic32_t *var, erts_aint32_t i)
-{
-#ifdef ERTS_SMP
- erts_atomic32_set_relb(var, i);
-#else
- *var = i;
-#endif
-}
-
-ERTS_GLB_INLINE void
-erts_smp_atomic32_dec_relb(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- erts_atomic32_dec_relb(decp);
-#else
- --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_dectest_relb(erts_smp_atomic32_t *decp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_dectest_relb(decp);
-#else
- return --(*decp);
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_acqb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_cmpxchg_acqb(xchgp, new, exp);
-#else
- erts_aint32_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE erts_aint32_t
-erts_smp_atomic32_cmpxchg_relb(erts_smp_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp)
-{
-#ifdef ERTS_SMP
- return erts_atomic32_cmpxchg_relb(xchgp, new, exp);
-#else
- erts_aint32_t old = *xchgp;
- if (old == exp)
- *xchgp = new;
- return old;
-#endif
-}
-
-ERTS_GLB_INLINE void
erts_smp_spinlock_init_x(erts_smp_spinlock_t *lock, char *name, Eterm extra)
{
#ifdef ERTS_SMP
@@ -1308,3 +1264,37 @@ erts_smp_thr_sigwait(const sigset_t *set, int *sig)
#endif /* #if ERTS_GLB_INLINE_INCL_FUNC_DEF */
#endif /* ERL_SMP_H */
+
+#ifdef ERTS_UNDEF_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#undef erts_smp_atomic_init
+#undef erts_smp_atomic_set
+#undef erts_smp_atomic_read
+#undef erts_smp_atomic_inctest
+#undef erts_smp_atomic_dectest
+#undef erts_smp_atomic_inc
+#undef erts_smp_atomic_dec
+#undef erts_smp_atomic_addtest
+#undef erts_smp_atomic_add
+#undef erts_smp_atomic_xchg
+#undef erts_smp_atomic_cmpxchg
+#undef erts_smp_atomic_bor
+#undef erts_smp_atomic_band
+
+#undef erts_smp_atomic32_init
+#undef erts_smp_atomic32_set
+#undef erts_smp_atomic32_read
+#undef erts_smp_atomic32_inctest
+#undef erts_smp_atomic32_dectest
+#undef erts_smp_atomic32_inc
+#undef erts_smp_atomic32_dec
+#undef erts_smp_atomic32_addtest
+#undef erts_smp_atomic32_add
+#undef erts_smp_atomic32_xchg
+#undef erts_smp_atomic32_cmpxchg
+#undef erts_smp_atomic32_bor
+#undef erts_smp_atomic32_band
+
+#endif
View
961 erts/emulator/beam/erl_threads.h
@@ -28,6 +28,11 @@
#define ERTS_SPIN_BODY ETHR_SPIN_BODY
#include "sys.h"
+
+typedef struct { SWord sint[2]; } erts_no_dw_atomic_t;
+typedef SWord erts_no_atomic_t;
+typedef Sint32 erts_no_atomic32_t;
+
#ifdef USE_THREADS
#define ETHR_TRY_INLINE_FUNCS
@@ -99,10 +104,12 @@ typedef ethr_rwmutex_opt erts_rwmtx_opt_t;
typedef ethr_tsd_key erts_tsd_key_t;
typedef ethr_ts_event erts_tse_t;
-typedef ethr_sint_t erts_aint_t;
-typedef ethr_atomic_t erts_atomic_t;
-typedef ethr_sint32_t erts_aint32_t;
-typedef ethr_atomic32_t erts_atomic32_t;
+#define erts_dw_aint_t ethr_dw_sint_t
+#define erts_dw_atomic_t ethr_dw_atomic_t
+#define erts_aint_t ethr_sint_t
+#define erts_atomic_t ethr_atomic_t
+#define erts_aint32_t ethr_sint32_t
+#define erts_atomic32_t ethr_atomic32_t
/* spinlock */
typedef struct {
@@ -164,10 +171,12 @@ typedef struct {
typedef int erts_rwmtx_t;
typedef int erts_tsd_key_t;
typedef int erts_tse_t;
-typedef SWord erts_aint_t;
-typedef SWord erts_atomic_t;
-typedef SWord erts_aint32_t;
-typedef SWord erts_atomic32_t;
+#define erts_dw_aint_t erts_no_dw_atomic_t
+#define erts_dw_atomic_t erts_no_dw_atomic_t
+#define erts_aint_t SWord
+#define erts_atomic_t erts_no_atomic_t
+#define erts_aint32_t Sint32
+#define erts_atomic32_t erts_no_atomic32_t
#if __GNUC__ > 2
typedef struct { } erts_spinlock_t;
typedef struct { } erts_rwlock_t;
@@ -247,65 +256,51 @@ ERTS_GLB_INLINE int erts_rwmtx_tryrwlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE void erts_rwmtx_rwunlock(erts_rwmtx_t *rwmtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rlocked(erts_rwmtx_t *mtx);
ERTS_GLB_INLINE int erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx);
-ERTS_GLB_INLINE void erts_atomic_init(erts_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE void erts_atomic_set(erts_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_read(erts_atomic_t *var);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_inctest(erts_atomic_t *incp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest(erts_atomic_t *decp);
-ERTS_GLB_INLINE void erts_atomic_inc(erts_atomic_t *incp);
-ERTS_GLB_INLINE void erts_atomic_dec(erts_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_addtest(erts_atomic_t *addp,
- erts_aint_t i);
-ERTS_GLB_INLINE void erts_atomic_add(erts_atomic_t *addp, erts_aint_t i);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_xchg(erts_atomic_t *xchgp,
- erts_aint_t new);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t expected);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_bor(erts_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_band(erts_atomic_t *var,
- erts_aint_t mask);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_read_acqb(erts_atomic_t *var);
-ERTS_GLB_INLINE void erts_atomic_set_relb(erts_atomic_t *var, erts_aint_t i);
-ERTS_GLB_INLINE void erts_atomic_dec_relb(erts_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_dectest_relb(erts_atomic_t *decp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_acqb(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE erts_aint_t erts_atomic_cmpxchg_relb(erts_atomic_t *xchgp,
- erts_aint_t new,
- erts_aint_t exp);
-ERTS_GLB_INLINE void erts_atomic32_init(erts_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE void erts_atomic32_set(erts_atomic32_t *var, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read(erts_atomic32_t *var);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_inctest(erts_atomic32_t *incp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest(erts_atomic32_t *decp);
-ERTS_GLB_INLINE void erts_atomic32_inc(erts_atomic32_t *incp);
-ERTS_GLB_INLINE void erts_atomic32_dec(erts_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_addtest(erts_atomic32_t *addp,
- erts_aint32_t i);
-ERTS_GLB_INLINE void erts_atomic32_add(erts_atomic32_t *addp, erts_aint32_t i);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_xchg(erts_atomic32_t *xchgp,
- erts_aint32_t new);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t expected);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_bor(erts_atomic32_t *var,
- erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_band(erts_atomic32_t *var,
- erts_aint32_t mask);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_read_acqb(erts_atomic32_t *var);
-ERTS_GLB_INLINE void erts_atomic32_set_relb(erts_atomic32_t *var,
- erts_aint32_t i);
-ERTS_GLB_INLINE void erts_atomic32_dec_relb(erts_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_dectest_relb(erts_atomic32_t *decp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_acqb(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
-ERTS_GLB_INLINE erts_aint32_t erts_atomic32_cmpxchg_relb(erts_atomic32_t *xchgp,
- erts_aint32_t new,
- erts_aint32_t exp);
+
+ERTS_GLB_INLINE void erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
+ERTS_GLB_INLINE void erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val);
+ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
+ erts_no_dw_atomic_t *val,
+ erts_no_dw_atomic_t *old_val);
+ERTS_GLB_INLINE void erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read(erts_no_atomic_t *var);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_inc_read(erts_no_atomic_t *incp);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_dec_read(erts_no_atomic_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic_inc(erts_no_atomic_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic_dec(erts_no_atomic_t *decp);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_add_read(erts_no_atomic_t *addp,
+ erts_aint_t i);
+ERTS_GLB_INLINE void erts_no_atomic_add(erts_no_atomic_t *addp, erts_aint_t i);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_bor(erts_no_atomic_t *var,
+ erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_read_band(erts_no_atomic_t *var,
+ erts_aint_t mask);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_xchg(erts_no_atomic_t *xchgp,
+ erts_aint_t new);
+ERTS_GLB_INLINE erts_aint_t erts_no_atomic_cmpxchg(erts_no_atomic_t *xchgp,
+ erts_aint_t new,
+ erts_aint_t expected);
+ERTS_GLB_INLINE void erts_no_atomic32_set(erts_no_atomic32_t *var,
+ erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read(erts_no_atomic32_t *var);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_inc_read(erts_no_atomic32_t *incp);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_dec_read(erts_no_atomic32_t *decp);
+ERTS_GLB_INLINE void erts_no_atomic32_inc(erts_no_atomic32_t *incp);
+ERTS_GLB_INLINE void erts_no_atomic32_dec(erts_no_atomic32_t *decp);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_add_read(erts_no_atomic32_t *addp,
+ erts_aint32_t i);
+ERTS_GLB_INLINE void erts_no_atomic32_add(erts_no_atomic32_t *addp,
+ erts_aint32_t i);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_bor(erts_no_atomic32_t *var,
+ erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_read_band(erts_no_atomic32_t *var,
+ erts_aint32_t mask);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_xchg(erts_no_atomic32_t *xchgp,
+ erts_aint32_t new);
+ERTS_GLB_INLINE erts_aint32_t erts_no_atomic32_cmpxchg(erts_no_atomic32_t *xchgp,
+ erts_aint32_t new,
+ erts_aint32_t expected);
+
ERTS_GLB_INLINE void erts_spinlock_init_x_opt(erts_spinlock_t *lock,
char *name,
Eterm extra,
@@ -362,6 +357,464 @@ ERTS_GLB_INLINE void erts_thr_sigmask(int how, const sigset_t *set,
ERTS_GLB_INLINE void erts_thr_sigwait(const sigset_t *set, int *sig);
#endif /* #ifdef HAVE_ETHR_SIG_FUNCS */
+/*
+ * Functions implementing atomic operations with with no (nob),
+ * full (mb), acquire (acqb), release (relb), read (rb), and
+ * write (wb) memory barriers.
+ *
+ * If thread support has been disabled, they are mapped to
+ * functions that performs the same operation, but aren't atomic
+ * and don't imply memory barriers.
+ */
+
+#ifdef USE_THREADS
+
+/* Double word size atomics */
+
+#define erts_dw_atomic_init_nob ethr_dw_atomic_init
+#define erts_dw_atomic_set_nob ethr_dw_atomic_set
+#define erts_dw_atomic_read_nob ethr_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_nob ethr_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_mb ethr_dw_atomic_init_mb
+#define erts_dw_atomic_set_mb ethr_dw_atomic_set_mb
+#define erts_dw_atomic_read_mb ethr_dw_atomic_read_mb
+#define erts_dw_atomic_cmpxchg_mb ethr_dw_atomic_cmpxchg_mb
+
+#define erts_dw_atomic_init_acqb ethr_dw_atomic_init_acqb
+#define erts_dw_atomic_set_acqb ethr_dw_atomic_set_acqb
+#define erts_dw_atomic_read_acqb ethr_dw_atomic_read_acqb
+#define erts_dw_atomic_cmpxchg_acqb ethr_dw_atomic_cmpxchg_acqb
+
+#define erts_dw_atomic_init_relb ethr_dw_atomic_init_relb
+#define erts_dw_atomic_set_relb ethr_dw_atomic_set_relb
+#define erts_dw_atomic_read_relb ethr_dw_atomic_read_relb
+#define erts_dw_atomic_cmpxchg_relb ethr_dw_atomic_cmpxchg_relb
+
+#define erts_dw_atomic_init_rb ethr_dw_atomic_init_rb
+#define erts_dw_atomic_set_rb ethr_dw_atomic_set_rb
+#define erts_dw_atomic_read_rb ethr_dw_atomic_read_rb
+#define erts_dw_atomic_cmpxchg_rb ethr_dw_atomic_cmpxchg_rb
+
+#define erts_dw_atomic_init_wb ethr_dw_atomic_init_wb
+#define erts_dw_atomic_set_wb ethr_dw_atomic_set_wb
+#define erts_dw_atomic_read_wb ethr_dw_atomic_read_wb
+#define erts_dw_atomic_cmpxchg_wb ethr_dw_atomic_cmpxchg_wb
+
+/* Word size atomics */
+
+#define erts_atomic_init_nob ethr_atomic_init
+#define erts_atomic_set_nob ethr_atomic_set
+#define erts_atomic_read_nob ethr_atomic_read
+#define erts_atomic_inc_read_nob ethr_atomic_inc_read
+#define erts_atomic_dec_read_nob ethr_atomic_dec_read
+#define erts_atomic_inc_nob ethr_atomic_inc
+#define erts_atomic_dec_nob ethr_atomic_dec
+#define erts_atomic_add_read_nob ethr_atomic_add_read
+#define erts_atomic_add_nob ethr_atomic_add
+#define erts_atomic_read_bor_nob ethr_atomic_read_bor
+#define erts_atomic_read_band_nob ethr_atomic_read_band
+#define erts_atomic_xchg_nob ethr_atomic_xchg
+#define erts_atomic_cmpxchg_nob ethr_atomic_cmpxchg
+
+#define erts_atomic_init_mb ethr_atomic_init_mb
+#define erts_atomic_set_mb ethr_atomic_set_mb
+#define erts_atomic_read_mb ethr_atomic_read_mb
+#define erts_atomic_inc_read_mb ethr_atomic_inc_read_mb
+#define erts_atomic_dec_read_mb ethr_atomic_dec_read_mb
+#define erts_atomic_inc_mb ethr_atomic_inc_mb
+#define erts_atomic_dec_mb ethr_atomic_dec_mb
+#define erts_atomic_add_read_mb ethr_atomic_add_read_mb
+#define erts_atomic_add_mb ethr_atomic_add_mb
+#define erts_atomic_read_bor_mb ethr_atomic_read_bor_mb
+#define erts_atomic_read_band_mb ethr_atomic_read_band_mb
+#define erts_atomic_xchg_mb ethr_atomic_xchg_mb
+#define erts_atomic_cmpxchg_mb ethr_atomic_cmpxchg_mb
+
+#define erts_atomic_init_acqb ethr_atomic_init_acqb
+#define erts_atomic_set_acqb ethr_atomic_set_acqb
+#define erts_atomic_read_acqb ethr_atomic_read_acqb
+#define erts_atomic_inc_read_acqb ethr_atomic_inc_read_acqb
+#define erts_atomic_dec_read_acqb ethr_atomic_dec_read_acqb
+#define erts_atomic_inc_acqb ethr_atomic_inc_acqb
+#define erts_atomic_dec_acqb ethr_atomic_dec_acqb
+#define erts_atomic_add_read_acqb ethr_atomic_add_read_acqb
+#define erts_atomic_add_acqb ethr_atomic_add_acqb
+#define erts_atomic_read_bor_acqb ethr_atomic_read_bor_acqb
+#define erts_atomic_read_band_acqb ethr_atomic_read_band_acqb
+#define erts_atomic_xchg_acqb ethr_atomic_xchg_acqb
+#define erts_atomic_cmpxchg_acqb ethr_atomic_cmpxchg_acqb
+
+#define erts_atomic_init_relb ethr_atomic_init_relb
+#define erts_atomic_set_relb ethr_atomic_set_relb
+#define erts_atomic_read_relb ethr_atomic_read_relb
+#define erts_atomic_inc_read_relb ethr_atomic_inc_read_relb
+#define erts_atomic_dec_read_relb ethr_atomic_dec_read_relb
+#define erts_atomic_inc_relb ethr_atomic_inc_relb
+#define erts_atomic_dec_relb ethr_atomic_dec_relb
+#define erts_atomic_add_read_relb ethr_atomic_add_read_relb
+#define erts_atomic_add_relb ethr_atomic_add_relb
+#define erts_atomic_read_bor_relb ethr_atomic_read_bor_relb
+#define erts_atomic_read_band_relb ethr_atomic_read_band_relb
+#define erts_atomic_xchg_relb ethr_atomic_xchg_relb
+#define erts_atomic_cmpxchg_relb ethr_atomic_cmpxchg_relb
+
+#define erts_atomic_init_rb ethr_atomic_init_rb
+#define erts_atomic_set_rb ethr_atomic_set_rb
+#define erts_atomic_read_rb ethr_atomic_read_rb
+#define erts_atomic_inc_read_rb ethr_atomic_inc_read_rb
+#define erts_atomic_dec_read_rb ethr_atomic_dec_read_rb
+#define erts_atomic_inc_rb ethr_atomic_inc_rb
+#define erts_atomic_dec_rb ethr_atomic_dec_rb
+#define erts_atomic_add_read_rb ethr_atomic_add_read_rb
+#define erts_atomic_add_rb ethr_atomic_add_rb
+#define erts_atomic_read_bor_rb ethr_atomic_read_bor_rb
+#define erts_atomic_read_band_rb ethr_atomic_read_band_rb
+#define erts_atomic_xchg_rb ethr_atomic_xchg_rb
+#define erts_atomic_cmpxchg_rb ethr_atomic_cmpxchg_rb
+
+#define erts_atomic_init_wb ethr_atomic_init_wb
+#define erts_atomic_set_wb ethr_atomic_set_wb
+#define erts_atomic_read_wb ethr_atomic_read_wb
+#define erts_atomic_inc_read_wb ethr_atomic_inc_read_wb
+#define erts_atomic_dec_read_wb ethr_atomic_dec_read_wb
+#define erts_atomic_inc_wb ethr_atomic_inc_wb
+#define erts_atomic_dec_wb ethr_atomic_dec_wb
+#define erts_atomic_add_read_wb ethr_atomic_add_read_wb
+#define erts_atomic_add_wb ethr_atomic_add_wb
+#define erts_atomic_read_bor_wb ethr_atomic_read_bor_wb
+#define erts_atomic_read_band_wb ethr_atomic_read_band_wb
+#define erts_atomic_xchg_wb ethr_atomic_xchg_wb
+#define erts_atomic_cmpxchg_wb ethr_atomic_cmpxchg_wb
+
+/* 32-bit atomics */
+
+#define erts_atomic32_init_nob ethr_atomic32_init
+#define erts_atomic32_set_nob ethr_atomic32_set
+#define erts_atomic32_read_nob ethr_atomic32_read
+#define erts_atomic32_inc_read_nob ethr_atomic32_inc_read
+#define erts_atomic32_dec_read_nob ethr_atomic32_dec_read
+#define erts_atomic32_inc_nob ethr_atomic32_inc
+#define erts_atomic32_dec_nob ethr_atomic32_dec
+#define erts_atomic32_add_read_nob ethr_atomic32_add_read
+#define erts_atomic32_add_nob ethr_atomic32_add
+#define erts_atomic32_read_bor_nob ethr_atomic32_read_bor
+#define erts_atomic32_read_band_nob ethr_atomic32_read_band
+#define erts_atomic32_xchg_nob ethr_atomic32_xchg
+#define erts_atomic32_cmpxchg_nob ethr_atomic32_cmpxchg
+
+#define erts_atomic32_init_mb ethr_atomic32_init_mb
+#define erts_atomic32_set_mb ethr_atomic32_set_mb
+#define erts_atomic32_read_mb ethr_atomic32_read_mb
+#define erts_atomic32_inc_read_mb ethr_atomic32_inc_read_mb
+#define erts_atomic32_dec_read_mb ethr_atomic32_dec_read_mb
+#define erts_atomic32_inc_mb ethr_atomic32_inc_mb
+#define erts_atomic32_dec_mb ethr_atomic32_dec_mb
+#define erts_atomic32_add_read_mb ethr_atomic32_add_read_mb
+#define erts_atomic32_add_mb ethr_atomic32_add_mb
+#define erts_atomic32_read_bor_mb ethr_atomic32_read_bor_mb
+#define erts_atomic32_read_band_mb ethr_atomic32_read_band_mb
+#define erts_atomic32_xchg_mb ethr_atomic32_xchg_mb
+#define erts_atomic32_cmpxchg_mb ethr_atomic32_cmpxchg_mb
+
+#define erts_atomic32_init_acqb ethr_atomic32_init_acqb
+#define erts_atomic32_set_acqb ethr_atomic32_set_acqb
+#define erts_atomic32_read_acqb ethr_atomic32_read_acqb
+#define erts_atomic32_inc_read_acqb ethr_atomic32_inc_read_acqb
+#define erts_atomic32_dec_read_acqb ethr_atomic32_dec_read_acqb
+#define erts_atomic32_inc_acqb ethr_atomic32_inc_acqb
+#define erts_atomic32_dec_acqb ethr_atomic32_dec_acqb
+#define erts_atomic32_add_read_acqb ethr_atomic32_add_read_acqb
+#define erts_atomic32_add_acqb ethr_atomic32_add_acqb
+#define erts_atomic32_read_bor_acqb ethr_atomic32_read_bor_acqb
+#define erts_atomic32_read_band_acqb ethr_atomic32_read_band_acqb
+#define erts_atomic32_xchg_acqb ethr_atomic32_xchg_acqb
+#define erts_atomic32_cmpxchg_acqb ethr_atomic32_cmpxchg_acqb
+
+#define erts_atomic32_init_relb ethr_atomic32_init_relb
+#define erts_atomic32_set_relb ethr_atomic32_set_relb
+#define erts_atomic32_read_relb ethr_atomic32_read_relb
+#define erts_atomic32_inc_read_relb ethr_atomic32_inc_read_relb
+#define erts_atomic32_dec_read_relb ethr_atomic32_dec_read_relb
+#define erts_atomic32_inc_relb ethr_atomic32_inc_relb
+#define erts_atomic32_dec_relb ethr_atomic32_dec_relb
+#define erts_atomic32_add_read_relb ethr_atomic32_add_read_relb
+#define erts_atomic32_add_relb ethr_atomic32_add_relb
+#define erts_atomic32_read_bor_relb ethr_atomic32_read_bor_relb
+#define erts_atomic32_read_band_relb ethr_atomic32_read_band_relb
+#define erts_atomic32_xchg_relb ethr_atomic32_xchg_relb
+#define erts_atomic32_cmpxchg_relb ethr_atomic32_cmpxchg_relb
+
+#define erts_atomic32_init_rb ethr_atomic32_init_rb
+#define erts_atomic32_set_rb ethr_atomic32_set_rb
+#define erts_atomic32_read_rb ethr_atomic32_read_rb
+#define erts_atomic32_inc_read_rb ethr_atomic32_inc_read_rb
+#define erts_atomic32_dec_read_rb ethr_atomic32_dec_read_rb
+#define erts_atomic32_inc_rb ethr_atomic32_inc_rb
+#define erts_atomic32_dec_rb ethr_atomic32_dec_rb
+#define erts_atomic32_add_read_rb ethr_atomic32_add_read_rb
+#define erts_atomic32_add_rb ethr_atomic32_add_rb
+#define erts_atomic32_read_bor_rb ethr_atomic32_read_bor_rb
+#define erts_atomic32_read_band_rb ethr_atomic32_read_band_rb
+#define erts_atomic32_xchg_rb ethr_atomic32_xchg_rb
+#define erts_atomic32_cmpxchg_rb ethr_atomic32_cmpxchg_rb
+
+#define erts_atomic32_init_wb ethr_atomic32_init_wb
+#define erts_atomic32_set_wb ethr_atomic32_set_wb
+#define erts_atomic32_read_wb ethr_atomic32_read_wb
+#define erts_atomic32_inc_read_wb ethr_atomic32_inc_read_wb
+#define erts_atomic32_dec_read_wb ethr_atomic32_dec_read_wb
+#define erts_atomic32_inc_wb ethr_atomic32_inc_wb
+#define erts_atomic32_dec_wb ethr_atomic32_dec_wb
+#define erts_atomic32_add_read_wb ethr_atomic32_add_read_wb
+#define erts_atomic32_add_wb ethr_atomic32_add_wb
+#define erts_atomic32_read_bor_wb ethr_atomic32_read_bor_wb
+#define erts_atomic32_read_band_wb ethr_atomic32_read_band_wb
+#define erts_atomic32_xchg_wb ethr_atomic32_xchg_wb
+#define erts_atomic32_cmpxchg_wb ethr_atomic32_cmpxchg_wb
+
+#else /* !USE_THREADS */
+
+/* Double word size atomics */
+
+#define erts_dw_atomic_init_nob erts_no_dw_atomic_set
+#define erts_dw_atomic_set_nob erts_no_dw_atomic_set
+#define erts_dw_atomic_read_nob erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_nob erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_mb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_mb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_mb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_mb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_acqb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_acqb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_acqb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_acqb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_relb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_relb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_relb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_relb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_rb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_rb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_rb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_rb erts_no_dw_atomic_cmpxchg
+
+#define erts_dw_atomic_init_wb erts_no_dw_atomic_init
+#define erts_dw_atomic_set_wb erts_no_dw_atomic_set
+#define erts_dw_atomic_read_wb erts_no_dw_atomic_read
+#define erts_dw_atomic_cmpxchg_wb erts_no_dw_atomic_cmpxchg
+
+/* Word size atomics */
+
+#define erts_atomic_init_nob erts_no_atomic_set
+#define erts_atomic_set_nob erts_no_atomic_set
+#define erts_atomic_read_nob erts_no_atomic_read
+#define erts_atomic_inc_read_nob erts_no_atomic_inc_read
+#define erts_atomic_dec_read_nob erts_no_atomic_dec_read
+#define erts_atomic_inc_nob erts_no_atomic_inc
+#define erts_atomic_dec_nob erts_no_atomic_dec
+#define erts_atomic_add_read_nob erts_no_atomic_add_read
+#define erts_atomic_add_nob erts_no_atomic_add
+#define erts_atomic_read_bor_nob erts_no_atomic_read_bor
+#define erts_atomic_read_band_nob erts_no_atomic_read_band
+#define erts_atomic_xchg_nob erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_nob erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_mb erts_no_atomic_set
+#define erts_atomic_set_mb erts_no_atomic_set
+#define erts_atomic_read_mb erts_no_atomic_read
+#define erts_atomic_inc_read_mb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_mb erts_no_atomic_dec_read
+#define erts_atomic_inc_mb erts_no_atomic_inc
+#define erts_atomic_dec_mb erts_no_atomic_dec
+#define erts_atomic_add_read_mb erts_no_atomic_add_read
+#define erts_atomic_add_mb erts_no_atomic_add
+#define erts_atomic_read_bor_mb erts_no_atomic_read_bor
+#define erts_atomic_read_band_mb erts_no_atomic_read_band
+#define erts_atomic_xchg_mb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_mb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_acqb erts_no_atomic_set
+#define erts_atomic_set_acqb erts_no_atomic_set
+#define erts_atomic_read_acqb erts_no_atomic_read
+#define erts_atomic_inc_read_acqb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_acqb erts_no_atomic_dec_read
+#define erts_atomic_inc_acqb erts_no_atomic_inc
+#define erts_atomic_dec_acqb erts_no_atomic_dec
+#define erts_atomic_add_read_acqb erts_no_atomic_add_read
+#define erts_atomic_add_acqb erts_no_atomic_add
+#define erts_atomic_read_bor_acqb erts_no_atomic_read_bor
+#define erts_atomic_read_band_acqb erts_no_atomic_read_band
+#define erts_atomic_xchg_acqb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_acqb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_relb erts_no_atomic_set
+#define erts_atomic_set_relb erts_no_atomic_set
+#define erts_atomic_read_relb erts_no_atomic_read
+#define erts_atomic_inc_read_relb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_relb erts_no_atomic_dec_read
+#define erts_atomic_inc_relb erts_no_atomic_inc
+#define erts_atomic_dec_relb erts_no_atomic_dec
+#define erts_atomic_add_read_relb erts_no_atomic_add_read
+#define erts_atomic_add_relb erts_no_atomic_add
+#define erts_atomic_read_bor_relb erts_no_atomic_read_bor
+#define erts_atomic_read_band_relb erts_no_atomic_read_band
+#define erts_atomic_xchg_relb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_relb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_rb erts_no_atomic_set
+#define erts_atomic_set_rb erts_no_atomic_set
+#define erts_atomic_read_rb erts_no_atomic_read
+#define erts_atomic_inc_read_rb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_rb erts_no_atomic_dec_read
+#define erts_atomic_inc_rb erts_no_atomic_inc
+#define erts_atomic_dec_rb erts_no_atomic_dec
+#define erts_atomic_add_read_rb erts_no_atomic_add_read
+#define erts_atomic_add_rb erts_no_atomic_add
+#define erts_atomic_read_bor_rb erts_no_atomic_read_bor
+#define erts_atomic_read_band_rb erts_no_atomic_read_band
+#define erts_atomic_xchg_rb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_rb erts_no_atomic_cmpxchg
+
+#define erts_atomic_init_wb erts_no_atomic_set
+#define erts_atomic_set_wb erts_no_atomic_set
+#define erts_atomic_read_wb erts_no_atomic_read
+#define erts_atomic_inc_read_wb erts_no_atomic_inc_read
+#define erts_atomic_dec_read_wb erts_no_atomic_dec_read
+#define erts_atomic_inc_wb erts_no_atomic_inc
+#define erts_atomic_dec_wb erts_no_atomic_dec
+#define erts_atomic_add_read_wb erts_no_atomic_add_read
+#define erts_atomic_add_wb erts_no_atomic_add
+#define erts_atomic_read_bor_wb erts_no_atomic_read_bor
+#define erts_atomic_read_band_wb erts_no_atomic_read_band
+#define erts_atomic_xchg_wb erts_no_atomic_xchg
+#define erts_atomic_cmpxchg_wb erts_no_atomic_cmpxchg
+
+/* 32-bit atomics */
+
+#define erts_atomic32_init_nob erts_no_atomic32_set
+#define erts_atomic32_set_nob erts_no_atomic32_set
+#define erts_atomic32_read_nob erts_no_atomic32_read
+#define erts_atomic32_inc_read_nob erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_nob erts_no_atomic32_dec_read
+#define erts_atomic32_inc_nob erts_no_atomic32_inc
+#define erts_atomic32_dec_nob erts_no_atomic32_dec
+#define erts_atomic32_add_read_nob erts_no_atomic32_add_read
+#define erts_atomic32_add_nob erts_no_atomic32_add
+#define erts_atomic32_read_bor_nob erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_nob erts_no_atomic32_read_band
+#define erts_atomic32_xchg_nob erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_nob erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_mb erts_no_atomic32_set
+#define erts_atomic32_set_mb erts_no_atomic32_set
+#define erts_atomic32_read_mb erts_no_atomic32_read
+#define erts_atomic32_inc_read_mb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_mb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_mb erts_no_atomic32_inc
+#define erts_atomic32_dec_mb erts_no_atomic32_dec
+#define erts_atomic32_add_read_mb erts_no_atomic32_add_read
+#define erts_atomic32_add_mb erts_no_atomic32_add
+#define erts_atomic32_read_bor_mb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_mb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_mb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_mb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_acqb erts_no_atomic32_set
+#define erts_atomic32_set_acqb erts_no_atomic32_set
+#define erts_atomic32_read_acqb erts_no_atomic32_read
+#define erts_atomic32_inc_read_acqb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_acqb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_acqb erts_no_atomic32_inc
+#define erts_atomic32_dec_acqb erts_no_atomic32_dec
+#define erts_atomic32_add_read_acqb erts_no_atomic32_add_read
+#define erts_atomic32_add_acqb erts_no_atomic32_add
+#define erts_atomic32_read_bor_acqb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_acqb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_acqb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_acqb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_relb erts_no_atomic32_set
+#define erts_atomic32_set_relb erts_no_atomic32_set
+#define erts_atomic32_read_relb erts_no_atomic32_read
+#define erts_atomic32_inc_read_relb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_relb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_relb erts_no_atomic32_inc
+#define erts_atomic32_dec_relb erts_no_atomic32_dec
+#define erts_atomic32_add_read_relb erts_no_atomic32_add_read
+#define erts_atomic32_add_relb erts_no_atomic32_add
+#define erts_atomic32_read_bor_relb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_relb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_relb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_relb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_rb erts_no_atomic32_set
+#define erts_atomic32_set_rb erts_no_atomic32_set
+#define erts_atomic32_read_rb erts_no_atomic32_read
+#define erts_atomic32_inc_read_rb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_rb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_rb erts_no_atomic32_inc
+#define erts_atomic32_dec_rb erts_no_atomic32_dec
+#define erts_atomic32_add_read_rb erts_no_atomic32_add_read
+#define erts_atomic32_add_rb erts_no_atomic32_add
+#define erts_atomic32_read_bor_rb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_rb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_rb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_rb erts_no_atomic32_cmpxchg
+
+#define erts_atomic32_init_wb erts_no_atomic32_set
+#define erts_atomic32_set_wb erts_no_atomic32_set
+#define erts_atomic32_read_wb erts_no_atomic32_read
+#define erts_atomic32_inc_read_wb erts_no_atomic32_inc_read
+#define erts_atomic32_dec_read_wb erts_no_atomic32_dec_read
+#define erts_atomic32_inc_wb erts_no_atomic32_inc
+#define erts_atomic32_dec_wb erts_no_atomic32_dec
+#define erts_atomic32_add_read_wb erts_no_atomic32_add_read
+#define erts_atomic32_add_wb erts_no_atomic32_add
+#define erts_atomic32_read_bor_wb erts_no_atomic32_read_bor
+#define erts_atomic32_read_band_wb erts_no_atomic32_read_band
+#define erts_atomic32_xchg_wb erts_no_atomic32_xchg
+#define erts_atomic32_cmpxchg_wb erts_no_atomic32_cmpxchg
+
+#endif /* !USE_THREADS */
+
+#ifndef ERTS_NO_DEPRECATED_ATOMICS
+
+/* Deprecated functions to replace */
+
+#define erts_atomic_init erts_atomic_init_nob
+#define erts_atomic_set erts_atomic_set_nob
+#define erts_atomic_read erts_atomic_read_nob
+#define erts_atomic_inctest erts_atomic_inc_read_mb
+#define erts_atomic_dectest erts_atomic_dec_read_mb
+#define erts_atomic_inc erts_atomic_inc_mb
+#define erts_atomic_dec erts_atomic_dec_mb
+#define erts_atomic_addtest erts_atomic_add_read_mb
+#define erts_atomic_add erts_atomic_add_mb
+#define erts_atomic_xchg erts_atomic_xchg_mb
+#define erts_atomic_cmpxchg erts_atomic_cmpxchg_mb
+#define erts_atomic_bor erts_atomic_read_bor_mb
+#define erts_atomic_band erts_atomic_read_band_mb
+
+#define erts_atomic32_init erts_atomic32_init_nob
+#define erts_atomic32_set erts_atomic32_set_nob
+#define erts_atomic32_read erts_atomic32_read_nob
+#define erts_atomic32_inctest erts_atomic32_inc_read_mb
+#define erts_atomic32_dectest erts_atomic32_dec_read_mb
+#define erts_atomic32_inc erts_atomic32_inc_mb
+#define erts_atomic32_dec erts_atomic32_dec_mb
+#define erts_atomic32_addtest erts_atomic32_add_read_mb
+#define erts_atomic32_add erts_atomic32_add_mb
+#define erts_atomic32_xchg erts_atomic32_xchg_mb
+#define erts_atomic32_cmpxchg erts_atomic32_cmpxchg_mb
+#define erts_atomic32_bor erts_atomic32_read_bor_mb
+#define erts_atomic32_band erts_atomic32_read_band_mb
+
+#endif
+
#if ERTS_GLB_INLINE_INCL_FUNC_DEF
ERTS_GLB_INLINE void
@@ -995,428 +1448,206 @@ erts_lc_rwmtx_is_rwlocked(erts_rwmtx_t *mtx)
#endif
}
+/* No atomic ops */
+
ERTS_GLB_INLINE void
-erts_atomic_init(erts_atomic_t *var, erts_aint_t i)
+erts_no_dw_atomic_set(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
{
-#ifdef USE_THREADS
- ethr_atomic_init(var, i);
-#else
- *var = i;
-#endif
+ var->sint[0] = val->sint[0];
+ var->sint[1] = val->sint[1];
}
ERTS_GLB_INLINE void
-erts_atomic_set(erts_atomic_t *var, erts_aint_t i)
+erts_no_dw_atomic_read(erts_no_dw_atomic_t *var, erts_no_dw_atomic_t *val)
+{
+ val->sint[0] = var->sint[0];
+ val->sint[1] = var->sint[1];
+}
+
+ERTS_GLB_INLINE int erts_no_dw_atomic_cmpxchg(erts_no_dw_atomic_t *var,
+ erts_no_dw_atomic_t *new_val,
+ erts_no_dw_atomic_t *old_val)
+{
+ if (var->sint[0] != old_val->sint[0] || var->sint[1] != old_val->sint[1]) {
+ erts_no_dw_atomic_read(var, old_val);
+ return 0;
+ }
+ else {
+ erts_no_dw_atomic_set(var, new_val);
+ return !0;
+ }
+}
+
+ERTS_GLB_INLINE void
+erts_no_atomic_set(erts_no_atomic_t *var, erts_aint_t i)