diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td index ea152bd130e9a..7919f43c86880 100644 --- a/llvm/include/llvm/Target/TargetSelectionDAG.td +++ b/llvm/include/llvm/Target/TargetSelectionDAG.td @@ -346,7 +346,7 @@ def SDTFPAtomic2 : SDTypeProfile<1, 2, [ ]>; def SDTAtomicStore : SDTypeProfile<0, 2, [ - SDTCisInt<0>, SDTCisPtrTy<1> + SDTCisPtrTy<1> ]>; def SDTAtomicLoad : SDTypeProfile<1, 1, [ SDTCisPtrTy<1> diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td index d8367412be587..082b93abf2e77 100644 --- a/llvm/lib/Target/X86/X86InstrAVX512.td +++ b/llvm/lib/Target/X86/X86InstrAVX512.td @@ -3356,8 +3356,9 @@ multiclass avx512_load_vl opc, string OpcodeStr, } multiclass avx512_store opc, string OpcodeStr, string BaseName, - X86VectorVTInfo _, PatFrag st_frag, PatFrag mstore, - X86SchedWriteMoveLS Sched, bit NoMRPattern = 0> { + X86VectorVTInfo _, SDPatternOperator st_frag, + PatFrag mstore, X86SchedWriteMoveLS Sched, + bit NoMRPattern = 0> { let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in { let isMoveReg = 1 in def rr_REV : AVX512PI; def extloadv8f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>; def extloadv16f16 : PatFrag<(ops node:$ptr), (extloadvf16 node:$ptr)>; -// Like 'store', but always requires vector size alignment. -def alignedstore : PatFrag<(ops node:$val, node:$ptr), - (store node:$val, node:$ptr), [{ - auto *St = cast(N); +// Like 'store' or 'atomic_store', but always requires vector size alignment. +def alignedstore : PatFrags<(ops node:$val, node:$ptr), + [(store node:$val, node:$ptr), + (atomic_store node:$val, node:$ptr)], [{ + auto *St = cast(N); return St->getAlign() >= St->getMemoryVT().getStoreSize(); }]> { let GISelPredicateCode = [{ diff --git a/llvm/test/CodeGen/X86/atomic-load-store.ll b/llvm/test/CodeGen/X86/atomic-load-store.ll index b2d2eab1e8a47..5be8d4d47fa9e 100644 --- a/llvm/test/CodeGen/X86/atomic-load-store.ll +++ b/llvm/test/CodeGen/X86/atomic-load-store.ll @@ -1191,6 +1191,92 @@ define void @store_atomic_vec4_bfloat(ptr %x, <4 x bfloat> %v) nounwind { ret void } +define void @store_atomic_vec4_float_align(ptr %x, <4 x float> %v) nounwind { +; CHECK-SSE2-O3-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE2-O3: # %bb.0: +; CHECK-SSE2-O3-NEXT: pushq %rax +; CHECK-SSE2-O3-NEXT: movq %xmm0, %rsi +; CHECK-SSE2-O3-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-SSE2-O3-NEXT: movq %xmm0, %rdx +; CHECK-SSE2-O3-NEXT: movl $3, %ecx +; CHECK-SSE2-O3-NEXT: callq __atomic_store_16@PLT +; CHECK-SSE2-O3-NEXT: popq %rax +; CHECK-SSE2-O3-NEXT: retq +; +; CHECK-SSE4-O3-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE4-O3: # %bb.0: +; CHECK-SSE4-O3-NEXT: pushq %rbx +; CHECK-SSE4-O3-NEXT: movdqa (%rdi), %xmm1 +; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm0, %rcx +; CHECK-SSE4-O3-NEXT: movq %xmm0, %rbx +; CHECK-SSE4-O3-NEXT: .p2align 4 +; CHECK-SSE4-O3-NEXT: .LBB39_1: # %atomicrmw.start +; CHECK-SSE4-O3-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-SSE4-O3-NEXT: movq %xmm1, %rax +; CHECK-SSE4-O3-NEXT: pextrq $1, %xmm1, %rdx +; CHECK-SSE4-O3-NEXT: lock cmpxchg16b (%rdi) +; CHECK-SSE4-O3-NEXT: movq %rdx, %xmm0 +; CHECK-SSE4-O3-NEXT: movq %rax, %xmm1 +; CHECK-SSE4-O3-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm0[0] +; CHECK-SSE4-O3-NEXT: jne .LBB39_1 +; CHECK-SSE4-O3-NEXT: # %bb.2: # %atomicrmw.end +; CHECK-SSE4-O3-NEXT: popq %rbx +; CHECK-SSE4-O3-NEXT: retq +; +; CHECK-AVX-O3-LABEL: store_atomic_vec4_float_align: +; CHECK-AVX-O3: # %bb.0: +; CHECK-AVX-O3-NEXT: vmovaps %xmm0, (%rdi) +; CHECK-AVX-O3-NEXT: retq +; +; CHECK-SSE2-O0-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE2-O0: # %bb.0: +; CHECK-SSE2-O0-NEXT: pushq %rax +; CHECK-SSE2-O0-NEXT: movq %xmm0, %rsi +; CHECK-SSE2-O0-NEXT: unpckhpd {{.*#+}} xmm0 = xmm0[1,1] +; CHECK-SSE2-O0-NEXT: movq %xmm0, %rdx +; CHECK-SSE2-O0-NEXT: movl $3, %ecx +; CHECK-SSE2-O0-NEXT: callq __atomic_store_16@PLT +; CHECK-SSE2-O0-NEXT: popq %rax +; CHECK-SSE2-O0-NEXT: retq +; +; CHECK-SSE4-O0-LABEL: store_atomic_vec4_float_align: +; CHECK-SSE4-O0: # %bb.0: +; CHECK-SSE4-O0-NEXT: pushq %rbx +; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-SSE4-O0-NEXT: movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill +; CHECK-SSE4-O0-NEXT: movaps (%rdi), %xmm0 +; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-SSE4-O0-NEXT: .LBB39_1: # %atomicrmw.start +; CHECK-SSE4-O0-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; CHECK-SSE4-O0-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload +; CHECK-SSE4-O0-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm1, %rcx +; CHECK-SSE4-O0-NEXT: movq %xmm1, %rbx +; CHECK-SSE4-O0-NEXT: movq %xmm0, %rax +; CHECK-SSE4-O0-NEXT: pextrq $1, %xmm0, %rdx +; CHECK-SSE4-O0-NEXT: lock cmpxchg16b (%rsi) +; CHECK-SSE4-O0-NEXT: movq %rax, %rcx +; CHECK-SSE4-O0-NEXT: sete %al +; CHECK-SSE4-O0-NEXT: movq %rdx, %xmm1 +; CHECK-SSE4-O0-NEXT: movq %rcx, %xmm0 +; CHECK-SSE4-O0-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0] +; CHECK-SSE4-O0-NEXT: testb $1, %al +; CHECK-SSE4-O0-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; CHECK-SSE4-O0-NEXT: jne .LBB39_2 +; CHECK-SSE4-O0-NEXT: jmp .LBB39_1 +; CHECK-SSE4-O0-NEXT: .LBB39_2: # %atomicrmw.end +; CHECK-SSE4-O0-NEXT: popq %rbx +; CHECK-SSE4-O0-NEXT: retq +; +; CHECK-AVX-O0-LABEL: store_atomic_vec4_float_align: +; CHECK-AVX-O0: # %bb.0: +; CHECK-AVX-O0-NEXT: vmovaps %xmm0, (%rdi) +; CHECK-AVX-O0-NEXT: retq + store atomic <4 x float> %v, ptr %x release, align 16 + ret void +} + define <2 x half> @atomic_vec2_half(ptr %x) { ; CHECK-SSE-O3-LABEL: atomic_vec2_half: ; CHECK-SSE-O3: # %bb.0: