sync/atomic: add Swap functions

Fixes #5722. R=golang-dev, khr, cshapiro, rsc, r CC=golang-dev https://golang.org/cl/12670045
golang · Aug 13, 2013 · 0e15b03 · 0e15b03
1 parent 39a7017
commit 0e15b03
Show file tree

Hide file tree

Showing 10 changed files with 548 additions and 45 deletions.
diff --git a/src/pkg/sync/atomic/64bit_arm.go b/src/pkg/sync/atomic/64bit_arm.go
@@ -34,3 +34,13 @@ func addUint64(val *uint64, delta uint64) (new uint64) {
 	}
 	return
 }
+
+func swapUint64(addr *uint64, new uint64) (old uint64) {
+	for {
+		old := *addr
+		if CompareAndSwapUint64(addr, old, new) {
+			break
+		}
+	}
+	return
+}
diff --git a/src/pkg/sync/atomic/asm_386.s b/src/pkg/sync/atomic/asm_386.s
@@ -6,6 +6,53 @@
 
 #include "../../../cmd/ld/textflag.h"
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0-12
+	JMP	·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0-12
+	MOVL	addr+0(FP), BP
+	MOVL	new+4(FP), AX
+	XCHGL	AX, 0(BP)
+	MOVL	AX, new+8(FP)
+	RET
+
+TEXT ·SwapInt64(SB),NOSPLIT,$0-20
+	JMP	·SwapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0-20
+	// no XCHGQ so use CMPXCHG8B loop
+	MOVL	addr+0(FP), BP
+	TESTL	$7, BP
+	JZ	2(PC)
+	MOVL	0, AX // crash with nil ptr deref
+	// CX:BX = new
+	MOVL	new_lo+4(FP), BX
+	MOVL	new_hi+8(FP), CX
+	// DX:AX = *addr
+	MOVL	0(BP), AX
+	MOVL	4(BP), DX
+swaploop:
+	// if *addr == DX:AX
+	//	*addr = CX:BX
+	// else
+	//	DX:AX = *addr
+	// all in one instruction
+	LOCK
+	CMPXCHG8B	0(BP)
+	JNZ	swaploop
+
+	// success
+	// return DX:AX
+	MOVL	AX, new_lo+12(FP)
+	MOVL	DX, new_hi+16(FP)
+	RET
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0-12
+	JMP	·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0-12
+	JMP	·SwapUint32(SB)
+
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-13
 	JMP	·CompareAndSwapUint32(SB)
 

diff --git a/src/pkg/sync/atomic/asm_amd64.s b/src/pkg/sync/atomic/asm_amd64.s
@@ -6,6 +6,32 @@
 
 #include "../../../cmd/ld/textflag.h"
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0-20
+	JMP	·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0-20
+	MOVQ	addr+0(FP), BP
+	MOVL	new+8(FP), AX
+	XCHGL	AX, 0(BP)
+	MOVL	AX, new+16(FP)
+	RET
+
+TEXT ·SwapInt64(SB),NOSPLIT,$0-24
+	JMP	·SwapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0-24
+	MOVQ	addr+0(FP), BP
+	MOVQ	new+8(FP), AX
+	XCHGQ	AX, 0(BP)
+	MOVQ	AX, new+16(FP)
+	RET
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0-24
+	JMP	·SwapUint64(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0-24
+	JMP	·SwapUint64(SB)
+
 TEXT ·CompareAndSwapInt32(SB),NOSPLIT,$0-17
 	JMP	·CompareAndSwapUint32(SB)
 

diff --git a/src/pkg/sync/atomic/asm_arm.s b/src/pkg/sync/atomic/asm_arm.s
@@ -91,6 +91,37 @@ add64loop:
 	MOVW	R5, rethi+16(FP)
 	RET
 
+TEXT ·armSwapUint32(SB),NOSPLIT,$0-12
+	MOVW	addr+0(FP), R1
+	MOVW	new+4(FP), R2
+swaploop:
+	// LDREX and STREX were introduced in ARM 6.
+	LDREX	(R1), R3
+	STREX	R2, (R1), R0
+	CMP	$0, R0
+	BNE	swaploop
+	MOVW	R3, old+8(FP)
+	RET
+
+TEXT ·armSwapUint64(SB),NOSPLIT,$0-20
+	BL	fastCheck64<>(SB)
+	MOVW	addr+0(FP), R1
+	// make unaligned atomic access panic
+	AND.S	$7, R1, R2
+	BEQ 	2(PC)
+	MOVW	R2, (R2)
+	MOVW	newlo+4(FP), R2
+	MOVW	newhi+8(FP), R3
+swap64loop:
+	// LDREXD and STREXD were introduced in ARM 11.
+	LDREXD	(R1), R4	// loads R4 and R5
+	STREXD	R2, (R1), R0	// stores R2 and R3
+	CMP	$0, R0
+	BNE	swap64loop
+	MOVW	R4, oldlo+12(FP)
+	MOVW	R5, oldhi+16(FP)
+	RET
+
 TEXT ·armLoadUint64(SB),NOSPLIT,$0-12
 	BL	fastCheck64<>(SB)
 	MOVW	addr+0(FP), R1

diff --git a/src/pkg/sync/atomic/asm_freebsd_arm.s b/src/pkg/sync/atomic/asm_freebsd_arm.s
@@ -28,6 +28,18 @@ TEXT ·AddUint32(SB),NOSPLIT,$0
 TEXT ·AddUintptr(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0
+	B ·armSwapUint32(SB)
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -40,6 +52,12 @@ TEXT ·AddInt64(SB),NOSPLIT,$0
 TEXT ·AddUint64(SB),NOSPLIT,$0
 	B ·addUint64(SB)
 
+TEXT ·SwapInt64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
 TEXT ·LoadInt32(SB),NOSPLIT,$0
 	B ·LoadUint32(SB)
 

diff --git a/src/pkg/sync/atomic/asm_linux_arm.s b/src/pkg/sync/atomic/asm_linux_arm.s
@@ -76,6 +76,26 @@ addloop1:
 TEXT ·AddUintptr(SB),NOSPLIT,$0
 	B	·AddUint32(SB)
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0
+	B	·SwapUint32(SB)
+
+// Implement using kernel cas for portability.
+TEXT ·SwapUint32(SB),NOSPLIT,$0-12
+	MOVW	addr+0(FP), R2
+	MOVW	new+4(FP), R1
+swaploop1:
+	MOVW	0(R2), R0
+	BL	cas<>(SB)
+	BCC	swaploop1
+	MOVW	R0, old+8(FP)
+	RET
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0
+	B	·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0
+	B	·SwapUint32(SB)
+
 TEXT cas64<>(SB),NOSPLIT,$0
 	MOVW	$0xffff0f60, PC // __kuser_cmpxchg64: Linux-3.1 and above
 
@@ -148,6 +168,12 @@ TEXT ·AddInt64(SB),NOSPLIT,$0
 TEXT ·AddUint64(SB),NOSPLIT,$0
 	B	·addUint64(SB)
 
+TEXT ·SwapInt64(SB),NOSPLIT,$0
+	B	·swapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0
+	B	·swapUint64(SB)
+
 TEXT ·LoadInt32(SB),NOSPLIT,$0
 	B	·LoadUint32(SB)
 

diff --git a/src/pkg/sync/atomic/asm_netbsd_arm.s b/src/pkg/sync/atomic/asm_netbsd_arm.s
@@ -28,6 +28,18 @@ TEXT ·AddUint32(SB),NOSPLIT,$0
 TEXT ·AddUintptr(SB),NOSPLIT,$0
 	B ·AddUint32(SB)
 
+TEXT ·SwapInt32(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapUint32(SB),NOSPLIT,$0
+	B ·armSwapUint32(SB)
+
+TEXT ·SwapUintptr(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
+TEXT ·SwapPointer(SB),NOSPLIT,$0
+	B ·SwapUint32(SB)
+
 TEXT ·CompareAndSwapInt64(SB),NOSPLIT,$0
 	B ·CompareAndSwapUint64(SB)
 
@@ -40,6 +52,12 @@ TEXT ·AddInt64(SB),NOSPLIT,$0
 TEXT ·AddUint64(SB),NOSPLIT,$0
 	B ·addUint64(SB)
 
+TEXT ·SwapInt64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
+TEXT ·SwapUint64(SB),NOSPLIT,$0
+	B ·swapUint64(SB)
+
 TEXT ·LoadInt32(SB),NOSPLIT,$0
 	B ·LoadUint32(SB)