[SystemZ] Enable AtomicExpand pass

llvm/lib/Target/SystemZ/SystemZISelLowering.cpp

@@ -872,6 +872,15 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const
   return false;
 }
 
+TargetLowering::AtomicExpansionKind
+SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const {
+  return (RMW->isFloatingPointOperation() ||
+          RMW->getOperation() == AtomicRMWInst::UIncWrap ||
+          RMW->getOperation() == AtomicRMWInst::UDecWrap)
+             ? AtomicExpansionKind::CmpXChg
+             : AtomicExpansionKind::None;
+}
+
 bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
   // We can use CGFI or CLGFI.
   return isInt<32>(Imm) || isUInt<32>(Imm);

llvm/lib/Target/SystemZ/SystemZISelLowering.h

@@ -452,6 +452,8 @@ class SystemZTargetLowering : public TargetLowering {
     return VT != MVT::f64;
   }
   bool hasInlineStackProbe(const MachineFunction &MF) const override;
+  AtomicExpansionKind
+  shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const override;
   bool isLegalICmpImmediate(int64_t Imm) const override;
   bool isLegalAddImmediate(int64_t Imm) const override;
   bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty,

llvm/lib/Target/SystemZ/SystemZTargetMachine.cpp

@@ -226,6 +226,8 @@ void SystemZPassConfig::addIRPasses() {
     addPass(createLoopDataPrefetchPass());
   }
 
+  addPass(createAtomicExpandPass());
+
   TargetPassConfig::addIRPasses();
 }
 

llvm/test/CodeGen/SystemZ/atomic-load-06.ll

@@ -0,0 +1,13 @@
+; Test float atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: lgf [[R:%r[0-9]+]], 0(%r2)
+; CHECK: sllg [[R]], [[R]], 32
+; CHECK: ldgr %f0, [[R]]
+; CHECK: br %r14
+  %val = load atomic float, ptr %src seq_cst, align 4
+  ret float %val
+}

llvm/test/CodeGen/SystemZ/atomic-load-07.ll

@@ -0,0 +1,11 @@
+; Test double atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: ld %f0, 0(%r2)
+; CHECK: br %r14
+  %val = load atomic double, ptr %src seq_cst, align 8
+  ret double %val
+}

llvm/test/CodeGen/SystemZ/atomic-load-08.ll

@@ -0,0 +1,20 @@
+; Test long double atomic loads. Expect a libcall.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: la %r4, 160(%r15)
+; CHECK: lghi %r2, 16
+; CHECK: lhi %r5, 5
+; CHECK: brasl %r14, __atomic_load@PLT
+; CHECK: ld [[FL:%f[0-9]+]], 160(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 168(%r15)
+; CHECK: std [[FL]], 0([[RET]])
+; CHECK: std [[FH]], 8([[RET]])
+; CHECK: br %r14
+  %val = load atomic fp128, ptr %src seq_cst, align 8
+  store fp128 %val, ptr %ret, align 8
+  ret void
+}

llvm/test/CodeGen/SystemZ/atomic-store-06.ll

@@ -0,0 +1,13 @@
+; Test float atomic loads.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %src, float %val) {
+; CHECK-LABEL: f1:
+; CHECK: lgdr [[R:%r[0-9]+]], %f0
+; CHECK: srlg [[R]], [[R]], 32
+; CHECK: st [[R]], 0(%r2)
+; CHECK: br %r14
+  store atomic float %val, ptr %src seq_cst, align 4
+  ret void
+}

llvm/test/CodeGen/SystemZ/atomic-store-07.ll

@@ -0,0 +1,11 @@
+; Test double atomic stores.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %dst, double %val) {
+; CHECK-LABEL: f1:
+; CHECK: std %f0, 0(%r2)
+; CHECK: br %r14
+  store atomic double %val, ptr %dst seq_cst, align 8
+  ret void
+}

llvm/test/CodeGen/SystemZ/atomic-store-08.ll

@@ -0,0 +1,20 @@
+; Test long double atomic stores. Expect a libcall.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %dst, ptr %src) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[FL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr %r3, %r2
+; CHECK: std [[FL]], 160(%r15)
+; CHECK: std [[FH]], 168(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: lghi %r2, 16
+; CHECK: lhi %r5, 5
+; CHECK: brasl %r14, __atomic_store@PLT
+; CHECK: br %r14
+  %val = load fp128, ptr %src, align 8
+  store atomic fp128 %val, ptr %dst seq_cst, align 8
+  ret void
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fadd-01.ll

@@ -0,0 +1,22 @@
+; Test atomic float addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: le [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: aebr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0(%r2)
+; CHECK: sllg [[RI]], [[RI]], 32
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[F]]
+; CHECK: br %r14
+  %res = atomicrmw fadd ptr %src, float %b seq_cst
+  ret float %res
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fadd-02.ll

@@ -0,0 +1,19 @@
+; Test atomic double addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: ld [[F:%f[0-9]+]], 0(%r2)
+; CHECK: [[L:\.L.+]]:
+; CHECK: lgdr [[RI:%r[0-9]+]], [[F]]
+; CHECK: adbr [[F]], %f0
+; CHECK: lgdr [[RO:%r[0-9]+]], [[F]]
+; CHECK: csg [[RI]], [[RO]], 0(%r2)
+; CHECK: ldgr [[F]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[F]]
+; CHECK: br %r14
+  %res = atomicrmw fadd ptr %src, double %b seq_cst
+  ret double %res
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fadd-03.ll

@@ -0,0 +1,30 @@
+; Test atomic long double addition. Expect a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: [[LABEL:\.L.+]]:
+; CHECK: lgdr [[RISH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RISL:%r[0-9]+]], [[FSL]]
+; CHECK: axbr [[FSL]], [[FBL]]
+; CHECK: lgdr [[ROSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[ROSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RISL]], [[ROSL]], 0(%r3)
+; CHECK: stg [[RISH]], 168(%r15)
+; CHECK: stg [[RISL]], 160(%r15)
+; CHECK: ld [[FSL]], 160(%r15)
+; CHECK: ld [[FSH]], 168(%r15)
+; CHECK: jl [[LABEL]]
+; CHECK: std [[FSL]], 0(%r2)
+; CHECK: std [[FSH]], 8(%r2)
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fadd ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fmax-01.ll

@@ -0,0 +1,27 @@
+; Test atomic float maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ler [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: ler %f2, [[FB]]
+; CHECK: brasl %r14, fmaxf@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
+; CHECK: sllg [[RO]], [[RI]], 32
+; CHECK: ldgr [[FSRC]], [[RO]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: br %r14
+  %res = atomicrmw fmax ptr %src, float %b seq_cst
+  ret float %res
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fmax-02.ll

@@ -0,0 +1,24 @@
+; Test atomic double maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[RB:%r[0-9]+]], %r2
+; CHECK: ld [[FB:%f[0-9]+]], 0(%r2)
+; CHECK: ldr [[FSRC:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ldr %f0, [[FB]]
+; CHECK: ldr %f2, [[FSRC]]
+; CHECK: brasl %r14, fmax@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FB]]
+; CHECK: csg [[RI]], [[RO]], 0([[RB]])
+; CHECK: ldgr [[FB]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[FB]]
+; CHECK: br %r14
+  %res = atomicrmw fmax ptr %src, double %b seq_cst
+  ret double %res
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fmax-03.ll

@@ -0,0 +1,42 @@
+; Test atomic long double maximum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r3
+; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: [[L:\.L.+]]:
+; CHECK: std [[FBL]], 160(%r15)
+; CHECK: std [[FBH]], 168(%r15)
+; CHECK: la %r2, 192(%r15)
+; CHECK: la %r3, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: std [[FSL]], 176(%r15)
+; CHECK: std [[FSH]], 184(%r15)
+; CHECK: brasl %r14, fmaxl@PLT
+; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
+; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
+; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
+; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
+; CHECK: stg [[RSH]], 216(%r15)
+; CHECK: stg [[RSL]], 208(%r15)
+; CHECK: ld [[FSL]], 208(%r15)
+; CHECK: ld [[FSH]], 216(%r15)
+; CHECK: jl [[L]]
+; CHECK: std [[FSL]], 0([[RET]])
+; CHECK: std [[FSH]], 8([[RET]])
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fmax ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fmin-01.ll

@@ -0,0 +1,27 @@
+; Test atomic float minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define float @f1(ptr %src, float %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: le [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ler [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: ler %f2, [[FB]]
+; CHECK: brasl %r14, fminf@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: srlg [[RO]], [[RO]], 32
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: srlg [[RI]], [[RI]], 32
+; CHECK: cs [[RI]], [[RO]], 0([[SRC]])
+; CHECK: sllg [[RO]], [[RI]], 32
+; CHECK: ldgr [[FSRC]], [[RO]]
+; CHECK: jl [[L]]
+; CHECK: ler %f0, [[FSRC]]
+; CHECK: br %r14
+  %res = atomicrmw fmin ptr %src, float %b seq_cst
+  ret float %res
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fmin-02.ll

@@ -0,0 +1,24 @@
+; Test atomic double minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define double @f1(ptr %src, double %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r2
+; CHECK: ld [[FSRC:%f[0-9]+]], 0(%r2)
+; CHECK: ldr [[FB:%f[0-9]+]], %f0
+; CHECK: [[L:\.L.+]]:
+; CHECK: ldr %f0, [[FSRC]]
+; CHECK: ldr %f2, [[FB]]
+; CHECK: brasl %r14, fmin@PLT
+; CHECK: lgdr [[RO:%r[0-9]+]], %f0
+; CHECK: lgdr [[RI:%r[0-9]+]], [[FSRC]]
+; CHECK: csg [[RI]], [[RO]], 0([[SRC]])
+; CHECK: ldgr [[FSRC]], [[RI]]
+; CHECK: jl [[L]]
+; CHECK: ldr %f0, [[FSRC]]
+; CHECK: br %r14
+  %res = atomicrmw fmin ptr %src, double %b seq_cst
+  ret double %res
+}

llvm/test/CodeGen/SystemZ/atomicrmw-fmin-03.ll

@@ -0,0 +1,42 @@
+; Test atomic long double minimum.
+; Expect a libcall in a compare-and-swap loop.
+;
+; RUN: llc < %s -mtriple=s390x-linux-gnu | FileCheck %s
+
+define void @f1(ptr %ret, ptr %src, ptr %b) {
+; CHECK-LABEL: f1:
+; CHECK: lgr [[SRC:%r[0-9]+]], %r3
+; CHECK: ld [[FBL:%f[0-9]+]], 0(%r4)
+; CHECK: ld [[FBH:%f[0-9]+]], 8(%r4)
+; CHECK: ld [[FSL:%f[0-9]+]], 0(%r3)
+; CHECK: ld [[FSH:%f[0-9]+]], 8(%r3)
+; CHECK: lgr [[RET:%r[0-9]+]], %r2
+; CHECK: [[L:\.L.+]]:
+; CHECK: std [[FBL]], 160(%r15)
+; CHECK: std [[FBH]], 168(%r15)
+; CHECK: la %r2, 192(%r15)
+; CHECK: la %r3, 176(%r15)
+; CHECK: la %r4, 160(%r15)
+; CHECK: std [[FSL]], 176(%r15)
+; CHECK: std [[FSH]], 184(%r15)
+; CHECK: brasl %r14, fminl@PLT
+; CHECK: ld [[FL:%f[0-9]+]], 192(%r15)
+; CHECK: ld [[FH:%f[0-9]+]], 200(%r15)
+; CHECK: lgdr [[RH:%r[0-9]+]], [[FH]]
+; CHECK: lgdr [[RL:%r[0-9]+]], [[FL]]
+; CHECK: lgdr [[RSH:%r[0-9]+]], [[FSH]]
+; CHECK: lgdr [[RSL:%r[0-9]+]], [[FSL]]
+; CHECK: cdsg [[RSL]], [[RL]], 0([[SRC]])
+; CHECK: stg [[RSH]], 216(%r15)
+; CHECK: stg [[RSL]], 208(%r15)
+; CHECK: ld [[FSL]], 208(%r15)
+; CHECK: ld [[FSH]], 216(%r15)
+; CHECK: jl [[L]]
+; CHECK: std [[FSL]], 0([[RET]])
+; CHECK: std [[FSH]], 8([[RET]])
+; CHECK: br %r14
+  %val = load fp128, ptr %b
+  %res = atomicrmw fmin ptr %src, fp128 %val seq_cst
+  store fp128 %res, ptr %ret
+  ret void
+}