Implement most AO_fetch_compare_and_swap primitives

* src/atomic_ops.c (AO_fetch_compare_and_swap_emulation): Implement.
* src/atomic_ops.h: Remove FIXME for AO_fetch_compare_and_swap.
* src/atomic_ops/generalize.h (AO_fetch_compare_and_swap_acquire,
AO_fetch_compare_and_swap_release, AO_fetch_compare_and_swap_acquire,
AO_fetch_compare_and_swap_write, AO_fetch_compare_and_swap_read,
AO_fetch_compare_and_swap, AO_fetch_compare_and_swap_full,
AO_fetch_compare_and_swap_release_write,
AO_fetch_compare_and_swap_acquire_read,
AO_fetch_compare_and_swap_dd_acquire_read): Implement; define the
corresponding AO_HAVE_... macro.
* src/atomic_ops/sysdeps/ao_t_is_int.h
(AO_int_fetch_compare_and_swap_full,
AO_int_fetch_compare_and_swap_acquire,
AO_int_fetch_compare_and_swap_release,
AO_int_fetch_compare_and_swap_write,
AO_int_fetch_compare_and_swap_read, AO_int_fetch_compare_and_swap):
Likewise.
* src/atomic_ops/sysdeps/armcc/arm_v6.h (AO_fetch_compare_and_swap):
Likewise.
* src/atomic_ops/sysdeps/emul_cas.h (AO_fetch_compare_and_swap_full):
Likewise.
* src/atomic_ops/sysdeps/gcc/arm.h (AO_fetch_compare_and_swap):
Likewise.
* src/atomic_ops/sysdeps/gcc/hexagon.h (AO_fetch_compare_and_swap):
Likewise.
* src/atomic_ops/sysdeps/gcc/ia64.h
(AO_fetch_compare_and_swap_acquire, AO_fetch_compare_and_swap_release,
AO_char_fetch_compare_and_swap_acquire,
AO_char_fetch_compare_and_swap_release,
AO_short_fetch_compare_and_swap_acquire,
AO_short_fetch_compare_and_swap_release,
AO_int_fetch_compare_and_swap_acquire,
AO_int_fetch_compare_and_swap_release): Likewise.
* src/atomic_ops/sysdeps/gcc/powerpc.h (AO_fetch_compare_and_swap,
AO_fetch_compare_and_swap_acquire, AO_fetch_compare_and_swap_release,
AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/gcc/x86.h (AO_fetch_compare_and_swap_full):
Likewise.
* src/atomic_ops/sysdeps/gcc/x86_64.h
(AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/generic_pthread.h
(AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/hpc/ia64.h
(AO_fetch_compare_and_swap_acquire, AO_fetch_compare_and_swap_release,
AO_char_fetch_compare_and_swap_acquire,
AO_char_fetch_compare_and_swap_release,
AO_short_fetch_compare_and_swap_acquire,
AO_short_fetch_compare_and_swap_release): Likewise.
* src/atomic_ops/sysdeps/icc/ia64.h
(AO_fetch_compare_and_swap_acquire, AO_fetch_compare_and_swap_release,
AO_char_fetch_compare_and_swap_acquire,
AO_char_fetch_compare_and_swap_release,
AO_short_fetch_compare_and_swap_acquire,
AO_short_fetch_compare_and_swap_release,
AO_int_fetch_compare_and_swap_acquire,
AO_int_fetch_compare_and_swap_release): Likewise.
* src/atomic_ops/sysdeps/msftc/common32_defs.h
(AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/msftc/x86_64.h
(AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/sunc/x86.h
(AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/sunc/x86_64.h
(AO_fetch_compare_and_swap_full): Likewise.
* src/atomic_ops/sysdeps/emul_cas.h
(AO_fetch_compare_and_swap_emulation): Declare the function.
* src/atomic_ops/sysdeps/gcc/mips.h: Adjust FIXME.
* src/atomic_ops/sysdeps/gcc/sparc.h: Likewise.
* src/atomic_ops/sysdeps/ibmc/powerpc.h (AO_fetch_compare_and_swap,
AO_fetch_compare_and_swap_acquire, AO_fetch_compare_and_swap_release,
AO_fetch_compare_and_swap_full): Add implementation stub.
* tests/list_atomic.template (list_atomicXX): Add
AO_fetch_compare_and_swapXX call.
* tests/test_atomic_include.template (test_atomicXX): Likewise.

src/atomic_ops.c

@@ -211,7 +211,26 @@ int AO_compare_and_swap_emulation(volatile AO_t *addr, AO_t old,
   return result;
 }
 
-/* FIXME: implement AO_fetch_compare_and_swap */
+AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
+                                         AO_t new_val)
+{
+  AO_TS_t *my_lock = AO_locks + AO_HASH(addr);
+  AO_t fetched_val;
+
+# ifndef AO_USE_NO_SIGNALS
+    sigset_t old_sigs;
+    block_all_signals(&old_sigs);
+# endif
+  lock(my_lock);
+  fetched_val = *addr;
+  if (fetched_val == old_val)
+    *addr = new_val;
+  unlock(my_lock);
+# ifndef AO_USE_NO_SIGNALS
+    sigprocmask(SIG_SETMASK, &old_sigs, NULL);
+# endif
+  return fetched_val;
+}
 
 int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
                                                 AO_t old_val1, AO_t old_val2,

src/atomic_ops.h

@@ -356,7 +356,6 @@
 # else
 #  error Cannot implement AO_compare_and_swap_full on this architecture.
 # endif
-/* FIXME: same for AO_fetch_compare_and_swap */
 #endif /* AO_REQUIRE_CAS && !AO_HAVE_compare_and_swap ... */
 
 /* The most common way to clear a test-and-set location         */

src/atomic_ops/generalize.h

@@ -1053,8 +1053,6 @@
 # endif
 #endif /* AO_HAVE_compare_and_swap_full */
 
-/* FIXME: implement AO_fetch_compare_and_swap */
-
 #if !defined(AO_HAVE_compare_and_swap) \
     && defined(AO_HAVE_compare_and_swap_release)
 # define AO_compare_and_swap(addr,old,new_val) \
@@ -1126,6 +1124,120 @@
 # endif
 #endif /* !AO_NO_DD_ORDERING */
 
+/* AO_fetch_compare_and_swap */
+#if defined(AO_HAVE_fetch_compare_and_swap) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_compare_and_swap_acquire)
+  AO_INLINE AO_t
+  AO_fetch_compare_and_swap_acquire(volatile AO_t *addr, AO_t old_val,
+                                    AO_t new_val)
+  {
+    AO_t result = AO_fetch_compare_and_swap(addr, old_val, new_val);
+    AO_nop_full();
+    return result;
+  }
+# define AO_HAVE_fetch_compare_and_swap_acquire
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap) && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_compare_and_swap_release)
+# define AO_fetch_compare_and_swap_release(addr,old_val,new_val) \
+              (AO_nop_full(), AO_fetch_compare_and_swap(addr,old_val,new_val))
+# define AO_HAVE_fetch_compare_and_swap_release
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap_full)
+# if !defined(AO_HAVE_fetch_compare_and_swap_release)
+#   define AO_fetch_compare_and_swap_release(addr,old_val,new_val) \
+                        AO_fetch_compare_and_swap_full(addr,old_val,new_val)
+#   define AO_HAVE_fetch_compare_and_swap_release
+# endif
+# if !defined(AO_HAVE_fetch_compare_and_swap_acquire)
+#   define AO_fetch_compare_and_swap_acquire(addr,old_val,new_val) \
+                        AO_fetch_compare_and_swap_full(addr,old_val,new_val)
+#   define AO_HAVE_fetch_compare_and_swap_acquire
+# endif
+# if !defined(AO_HAVE_fetch_compare_and_swap_write)
+#   define AO_fetch_compare_and_swap_write(addr,old_val,new_val) \
+                        AO_fetch_compare_and_swap_full(addr,old_val,new_val)
+#   define AO_HAVE_fetch_compare_and_swap_write
+# endif
+# if !defined(AO_HAVE_fetch_compare_and_swap_read)
+#   define AO_fetch_compare_and_swap_read(addr,old_val,new_val) \
+                        AO_fetch_compare_and_swap_full(addr,old_val,new_val)
+#   define AO_HAVE_fetch_compare_and_swap_read
+# endif
+#endif /* AO_HAVE_fetch_compare_and_swap_full */
+
+#if !defined(AO_HAVE_fetch_compare_and_swap) \
+    && defined(AO_HAVE_fetch_compare_and_swap_release)
+# define AO_fetch_compare_and_swap(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_release(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap
+#endif
+#if !defined(AO_HAVE_fetch_compare_and_swap) \
+    && defined(AO_HAVE_fetch_compare_and_swap_acquire)
+# define AO_fetch_compare_and_swap(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_acquire(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap
+#endif
+#if !defined(AO_HAVE_fetch_compare_and_swap) \
+    && defined(AO_HAVE_fetch_compare_and_swap_write)
+# define AO_fetch_compare_and_swap(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_write(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap
+#endif
+#if !defined(AO_HAVE_fetch_compare_and_swap) \
+    && defined(AO_HAVE_fetch_compare_and_swap_read)
+# define AO_fetch_compare_and_swap(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_read(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap
+#endif
+
+#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
+    && defined(AO_HAVE_nop_full) \
+    && !defined(AO_HAVE_fetch_compare_and_swap_full)
+# define AO_fetch_compare_and_swap_full(addr,old_val,new_val) \
+      (AO_nop_full(), AO_fetch_compare_and_swap_acquire(addr,old_val,new_val))
+# define AO_HAVE_fetch_compare_and_swap_full
+#endif
+
+#if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
+    && defined(AO_HAVE_fetch_compare_and_swap_write)
+# define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_write(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap_release_write
+#endif
+#if !defined(AO_HAVE_fetch_compare_and_swap_release_write) \
+    && defined(AO_HAVE_fetch_compare_and_swap_release)
+# define AO_fetch_compare_and_swap_release_write(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_release(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap_release_write
+#endif
+#if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
+    && defined(AO_HAVE_fetch_compare_and_swap_read)
+# define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_read(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap_acquire_read
+#endif
+#if !defined(AO_HAVE_fetch_compare_and_swap_acquire_read) \
+    && defined(AO_HAVE_fetch_compare_and_swap_acquire)
+# define AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_acquire(addr,old_val,new_val)
+# define AO_HAVE_fetch_compare_and_swap_acquire_read
+#endif
+
+#ifdef AO_NO_DD_ORDERING
+# if defined(AO_HAVE_fetch_compare_and_swap_acquire_read)
+#   define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap_acquire_read(addr,old_val,new_val)
+#   define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
+# endif
+#else
+# if defined(AO_HAVE_fetch_compare_and_swap)
+#   define AO_fetch_compare_and_swap_dd_acquire_read(addr,old_val,new_val) \
+                AO_fetch_compare_and_swap(addr,old_val,new_val)
+#   define AO_HAVE_fetch_compare_and_swap_dd_acquire_read
+# endif
+#endif /* !AO_NO_DD_ORDERING */
+
 #include "generalize-small.h"
 
 /* Compare_double_and_swap_double */

src/atomic_ops/sysdeps/ao_t_is_int.h

@@ -72,7 +72,49 @@
 # define AO_HAVE_int_compare_and_swap
 #endif
 
-/* FIXME: implement AO_fetch_compare_and_swap */
+/* AO_fetch_compare_and_swap */
+#if defined(AO_HAVE_fetch_compare_and_swap_full) \
+    && !defined(AO_HAVE_int_fetch_compare_and_swap_full)
+# define AO_int_fetch_compare_and_swap_full(addr, old, new_val) \
+       ((int)AO_fetch_compare_and_swap_full((volatile AO_t *)(addr), \
+                                            (AO_t)(old), (AO_t)(new_val)))
+# define AO_HAVE_int_fetch_compare_and_swap_full
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap_acquire) \
+    && !defined(AO_HAVE_int_fetch_compare_and_swap_acquire)
+# define AO_int_fetch_compare_and_swap_acquire(addr, old, new_val) \
+       ((int)AO_fetch_compare_and_swap_acquire((volatile AO_t *)(addr), \
+                                               (AO_t)(old), (AO_t)(new_val)))
+# define AO_HAVE_int_fetch_compare_and_swap_acquire
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap_release) \
+    && !defined(AO_HAVE_int_fetch_compare_and_swap_release)
+# define AO_int_fetch_compare_and_swap_release(addr, old, new_val) \
+       ((int)AO_fetch_compare_and_swap_release((volatile AO_t *)(addr), \
+                                               (AO_t)(old), (AO_t)(new_val)))
+# define AO_HAVE_int_fetch_compare_and_swap_release
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap_write) \
+    && !defined(AO_HAVE_int_fetch_compare_and_swap_write)
+# define AO_int_fetch_compare_and_swap_write(addr, old, new_val) \
+       ((int)AO_fetch_compare_and_swap_write((volatile AO_t *)(addr), \
+                                             (AO_t)(old), (AO_t)(new_val)))
+# define AO_HAVE_int_fetch_compare_and_swap_write
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap_read) \
+    && !defined(AO_HAVE_int_fetch_compare_and_swap_read)
+# define AO_int_fetch_compare_and_swap_read(addr, old, new_val) \
+       ((int)AO_fetch_compare_and_swap_read((volatile AO_t *)(addr), \
+                                            (AO_t)(old), (AO_t)(new_val)))
+# define AO_HAVE_int_fetch_compare_and_swap_read
+#endif
+#if defined(AO_HAVE_fetch_compare_and_swap) \
+    && !defined(AO_HAVE_int_fetch_compare_and_swap)
+# define AO_int_fetch_compare_and_swap(addr, old, new_val) \
+       ((int)AO_fetch_compare_and_swap((volatile AO_t *)(addr), \
+                                       (AO_t)(old), (AO_t)(new_val)))
+# define AO_HAVE_int_fetch_compare_and_swap
+#endif
 
 /* AO_load */
 #if defined(AO_HAVE_load_acquire) \

src/atomic_ops/sysdeps/armcc/arm_v6.h

@@ -198,7 +198,26 @@ __asm__ {
 }
 #define AO_HAVE_compare_and_swap
 
-/* FIXME: implement AO_fetch_compare_and_swap */
+AO_INLINE AO_t
+AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
+{
+         AO_t fetched_val, tmp;
+
+retry:
+__asm__ {
+        mov     tmp, #2
+        ldrex   fetched_val, [addr]
+        teq     fetched_val, old_val
+#     ifdef __thumb__
+        it      eq
+#     endif
+        strexeq tmp, new_val, [addr]
+        teq     tmp, #1
+        beq     retry
+        }
+        return fetched_val;
+}
+#define AO_HAVE_fetch_compare_and_swap
 
 /* helper functions for the Realview compiler: LDREXD is not usable
  * with inline assembler, so use the "embedded" assembler as

src/atomic_ops/sysdeps/emul_cas.h

@@ -46,6 +46,9 @@
 int AO_compare_and_swap_emulation(volatile AO_t *addr, AO_t old,
                                   AO_t new_val);
 
+AO_t AO_fetch_compare_and_swap_emulation(volatile AO_t *addr, AO_t old_val,
+                                         AO_t new_val);
+
 int AO_compare_double_and_swap_double_emulation(volatile AO_double_t *addr,
                                                 AO_t old_val1, AO_t old_val2,
                                                 AO_t new_val1, AO_t new_val2);
@@ -56,6 +59,12 @@ void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
        AO_compare_and_swap_emulation(addr, old, newval)
 #define AO_HAVE_compare_and_swap_full
 
+#ifndef AO_HAVE_fetch_compare_and_swap_full
+# define AO_fetch_compare_and_swap_full(addr, old, newval) \
+                AO_fetch_compare_and_swap_emulation(addr, old, newval)
+# define AO_HAVE_fetch_compare_and_swap_full
+#endif
+
 #ifndef AO_HAVE_compare_double_and_swap_double
 # define AO_compare_double_and_swap_double_full(addr, old1, old2, \
                                                 newval1, newval2) \
@@ -64,8 +73,6 @@ void AO_store_full_emulation(volatile AO_t *addr, AO_t val);
 # define AO_HAVE_compare_double_and_swap_double_full
 #endif
 
-/* FIXME: implement AO_fetch_compare_and_swap */
-
 #undef AO_store
 #undef AO_HAVE_store
 #undef AO_store_write

src/atomic_ops/sysdeps/gcc/arm.h

@@ -265,7 +265,29 @@ AO_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
 }
 #define AO_HAVE_compare_and_swap
 
-/* FIXME: implement AO_fetch_compare_and_swap */
+AO_INLINE AO_t
+AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
+{
+  AO_t fetched_val, flag;
+
+  __asm__ __volatile__("@AO_fetch_compare_and_swap\n"
+    AO_THUMB_GO_ARM
+    "1:     mov     %0, #2\n"           /* store a flag */
+    "       ldrex   %1, [%3]\n"         /* get original */
+    "       teq     %1, %4\n"           /* see if match */
+#   ifdef __thumb2__
+      "       it      eq\n"
+#   endif
+    "       strexeq %0, %5, [%3]\n"     /* store new one if matched */
+    "       teq     %0, #1\n"
+    "       beq     1b\n"               /* if update failed, repeat */
+    AO_THUMB_RESTORE_MODE
+    : "=&r"(flag), "=&r"(fetched_val), "+m"(*addr)
+    : "r"(addr), "r"(old_val), "r"(new_val)
+    : AO_THUMB_SWITCH_CLOBBERS "cc");
+  return fetched_val;
+}
+#define AO_HAVE_fetch_compare_and_swap
 
 #if !defined(__ARM_ARCH_6__) && !defined(__ARM_ARCH_6J__) \
     && !defined(__ARM_ARCH_6T2__) && !defined(__ARM_ARCH_6Z__) \

src/atomic_ops/sysdeps/gcc/hexagon.h

@@ -92,6 +92,27 @@ AO_compare_and_swap(volatile AO_t *addr, AO_t old, AO_t new_val)
 }
 #define AO_HAVE_compare_and_swap
 
-/* FIXME: implement AO_fetch_compare_and_swap */
+AO_INLINE AO_t
+AO_fetch_compare_and_swap(volatile AO_t *addr, AO_t old_val, AO_t new_val)
+{
+  AO_t __oldval;
+
+  __asm__ __volatile__(
+     "1:\n"
+     "  %0 = memw_locked(%2);\n"        /* load and reserve            */
+     "  {\n"
+     "    p2 = cmp.eq(%0,%3);\n"        /* if load is not equal to     */
+     "    if (!p2.new) jump:nt 2f; \n"  /* old_val, fail               */
+     "  }\n"
+     "  memw_locked(%2,p1) = %4;\n"     /* else store conditional      */
+     "  if (!p1) jump 1b;\n"            /* retry if lost reservation   */
+     "2:\n"
+     : "=&r" (__oldval), "+m"(*addr)
+     : "r" (addr), "r" (old_val), "r" (new_val)
+     : "p1", "p2", "memory"
+  );
+  return __oldval;
+}
+#define AO_HAVE_fetch_compare_and_swap
 
 #include "../ao_t_is_int.h"