Skip to content

Commit 6a19a10

Browse files
hjelmnhjelmn
committed
atomic/ppc: add atomics for load-link, store-conditional, and swap
This commit adds implementations of opal_atomic_ll_32/64 and opal_atomic_sc_32/64. These atomics can be used to implement more efficient lifo/fifo operations on supported platforms. The only supported platform with this commit is powerpc/power. This commit also adds an implementation of opal_atomic_swap_32/64 for powerpc. Tested with Power8. Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
1 parent f87dbca commit 6a19a10

File tree

3 files changed

+121
-3
lines changed

3 files changed

+121
-3
lines changed

‎opal/include/opal/sys/atomic.h‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -11,7 +12,7 @@
1112
* All rights reserved.
1213
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
1314
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
14-
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
15+
* Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
1516
* reserved.
1617
* $COPYRIGHT$
1718
*
@@ -38,7 +39,7 @@
3839
* - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
3940
* - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
4041
* - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/cmpset can be done "atomicly"
41-
* - \c OPAL_HAVE_ATOMIC_MATH_64 if 32 bit add/sub/cmpset can be done "atomicly"
42+
* - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/cmpset can be done "atomicly"
4243
*
4344
* Note that for the Atomic math, atomic add/sub may be implemented as
4445
* C code using opal_atomic_cmpset. The appearance of atomic
@@ -177,6 +178,12 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
177178
#ifndef OPAL_HAVE_ATOMIC_CMPSET_128
178179
#define OPAL_HAVE_ATOMIC_CMPSET_128 0
179180
#endif
181+
#ifndef OPAL_HAVE_ATOMIC_LLSC_32
182+
#define OPAL_HAVE_ATOMIC_LLSC_32 0
183+
#endif
184+
#ifndef OPAL_HAVE_ATOMIC_LLSC_64
185+
#define OPAL_HAVE_ATOMIC_LLSC_64 0
186+
#endif
180187
#endif /* DOXYGEN */
181188

182189
/**********************************************************************

‎opal/include/opal/sys/atomic_impl.h‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
* Copyright (c) 2004-2005 The Regents of the University of California.
1212
* All rights reserved.
1313
* Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved.
14-
* Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
14+
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
1515
* reserved.
1616
* $COPYRIGHT$
1717
*
@@ -274,6 +274,30 @@ static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr,
274274

275275
#endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */
276276

277+
#if (OPAL_HAVE_ATOMIC_LLSC_32 || OPAL_HAVE_ATOMIC_LLSC_64)
278+
279+
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32
280+
281+
#define opal_atomic_ll_ptr(addr) (void *) opal_atomic_ll_32((int32_t *) addr)
282+
#define opal_atomic_sc_ptr(addr, newval) opal_atomic_sc_32((int32_t *) addr, (int32_t) newval)
283+
284+
#define OPAL_HAVE_ATOMIC_LLSC_PTR 1
285+
286+
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64
287+
288+
#define opal_atomic_ll_ptr(addr) (void *) opal_atomic_ll_64((int64_t *) addr)
289+
#define opal_atomic_sc_ptr(addr, newval) opal_atomic_sc_64((int64_t *) addr, (int64_t) newval)
290+
291+
#define OPAL_HAVE_ATOMIC_LLSC_PTR 1
292+
293+
#endif
294+
295+
#endif /* (OPAL_HAVE_ATOMIC_LLSC_32 || OPAL_HAVE_ATOMIC_LLSC_64)*/
296+
297+
#if !defined(OPAL_HAVE_ATOMIC_LLSC_PTR)
298+
#define OPAL_HAVE_ATOMIC_LLSC_PTR 0
299+
#endif
300+
277301
#if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64
278302

279303

‎opal/include/opal/sys/powerpc/atomic.h‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
12
/*
23
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
34
* University Research and Technology
@@ -10,6 +11,8 @@
1011
* Copyright (c) 2004-2005 The Regents of the University of California.
1112
* All rights reserved.
1213
* Copyright (c) 2010 IBM Corporation. All rights reserved.
14+
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
15+
* reserved.
1316
* $COPYRIGHT$
1417
*
1518
* Additional copyrights may follow
@@ -40,6 +43,8 @@
4043
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
4144

4245
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
46+
#define OPAL_HAVE_ATOMIC_SWAP_32 1
47+
#define OPAL_HAVE_ATOMIC_LLSC_32 1
4348

4449
#define OPAL_HAVE_ATOMIC_MATH_32 1
4550
#define OPAL_HAVE_ATOMIC_ADD_32 1
@@ -48,6 +53,8 @@
4853

4954
#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
5055
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
56+
#define OPAL_HAVE_ATOMIC_SWAP_64 1
57+
#define OPAL_HAVE_ATOMIC_LLSC_64 1
5158
#endif
5259

5360

@@ -140,6 +147,32 @@ static inline int opal_atomic_cmpset_32(volatile int32_t *addr,
140147
return (ret == oldval);
141148
}
142149

150+
static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
151+
{
152+
int32_t ret;
153+
154+
__asm__ __volatile__ ("lwarx %0, 0, %1 \n\t"
155+
: "=&r" (ret)
156+
: "r" (addr)
157+
:);
158+
return ret;
159+
}
160+
161+
static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
162+
{
163+
int32_t ret, foo;
164+
165+
__asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t"
166+
" li %0,0 \n\t"
167+
" bne- 1f \n\t"
168+
" ori %0,%0,1 \n\t"
169+
"1:"
170+
: "=r" (ret), "=m" (*addr), "=r" (foo)
171+
: "r" (addr), "r" (newval)
172+
: "cc", "memory");
173+
return ret;
174+
}
175+
143176
/* these two functions aren't inlined in the non-gcc case because then
144177
there would be two function calls (since neither cmpset_32 nor
145178
atomic_?mb can be inlined). Instead, we "inline" them by hand in
@@ -164,6 +197,20 @@ static inline int opal_atomic_cmpset_rel_32(volatile int32_t *addr,
164197
return opal_atomic_cmpset_32(addr, oldval, newval);
165198
}
166199

200+
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
201+
{
202+
int32_t ret;
203+
204+
__asm__ __volatile__ ("1: lwarx %0, 0, %2 \n\t"
205+
" stwcx. %3, 0, %2 \n\t"
206+
" bne- 1b \n\t"
207+
: "=&r" (ret), "=m" (*addr)
208+
: "r" (addr), "r" (newval)
209+
: "cc", "memory");
210+
211+
return ret;
212+
}
213+
167214
#endif /* OPAL_GCC_INLINE_ASSEMBLY */
168215

169216

@@ -189,6 +236,32 @@ static inline int opal_atomic_cmpset_64(volatile int64_t *addr,
189236
return (ret == oldval);
190237
}
191238

239+
static inline int64_t opal_atomic_ll_64(volatile int64_t *addr)
240+
{
241+
int64_t ret;
242+
243+
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t"
244+
: "=&r" (ret)
245+
: "r" (addr)
246+
:);
247+
return ret;
248+
}
249+
250+
static inline int opal_atomic_sc_64(volatile int64_t *addr, int64_t newval)
251+
{
252+
int32_t ret, foo;
253+
254+
__asm__ __volatile__ (" stdcx. %4, 0, %3 \n\t"
255+
" li %0,0 \n\t"
256+
" bne- 1f \n\t"
257+
" ori %0,%0,1 \n\t"
258+
"1:"
259+
: "=r" (ret), "=m" (*addr), "=r" (foo)
260+
: "r" (addr), "r" (newval)
261+
: "cc", "memory");
262+
return ret;
263+
}
264+
192265
/* these two functions aren't inlined in the non-gcc case because then
193266
there would be two function calls (since neither cmpset_64 nor
194267
atomic_?mb can be inlined). Instead, we "inline" them by hand in
@@ -213,6 +286,20 @@ static inline int opal_atomic_cmpset_rel_64(volatile int64_t *addr,
213286
return opal_atomic_cmpset_64(addr, oldval, newval);
214287
}
215288

289+
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval)
290+
{
291+
int64_t ret;
292+
293+
__asm__ __volatile__ ("1: ldarx %0, 0, %2 \n\t"
294+
" stdcx. %3, 0, %2 \n\t"
295+
" bne- 1b \n\t"
296+
: "=&r" (ret), "=m" (*addr)
297+
: "r" (addr), "r" (newval)
298+
: "cc", "memory");
299+
300+
return ret;
301+
}
302+
216303
#endif /* OPAL_GCC_INLINE_ASSEMBLY */
217304

218305
#elif (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC32) && OPAL_ASM_SUPPORT_64BIT

0 commit comments

Comments
 (0)