Skip to content

Commit

Permalink
[ARM] Match dual lane vmovs from insert_vector_elt
Browse files Browse the repository at this point in the history 
MVE has a dual lane vector move instruction, capable of moving two
general purpose registers into lanes of a vector register. They look
like one of:
  vmov q0[2], q0[0], r2, r0
  vmov q0[3], q0[1], r3, r1
They only accept these lane indices though (and only insert into an
i32), either moving lanes 1 and 3, or 0 and 2.

This patch adds some tablegen patterns for them, selecting from vector
inserts elements. Because the insert_elements are know to be
canonicalized to ascending order there are several patterns that we need
to select. These lane indices are:

3 2 1 0    -> vmovqrr 31; vmovqrr 20
3 2 1      -> vmovqrr 31; vmov 2
3 1        -> vmovqrr 31
2 1 0      -> vmovqrr 20; vmov 1
2 0        -> vmovqrr 20

With the top one being the most common. All other potential patterns of
lane indices will be matched by a combination of these and the
individual vmov pattern already present. This does mean that we are
selecting several machine instructions at once due to the need to
re-arrange the inserts, but in this case there is nothing else that will
attempt to match an insert_vector_elt node.

This is a recommit of 6cc3d80 after
fixing the backward instruction definitions.
  • Loading branch information
@davemgreen
davemgreen committed Dec 18, 2020
1 parent 4652718 commit e1c1adf
Show file tree
Hide file tree
Showing 54 changed files with 5,693 additions and 6,874 deletions.
8 changes: 8 additions & 0 deletions llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4790,6 +4790,14 @@ bool ARMBaseInstrInfo::verifyInstruction(const MachineInstr &MI,
}
}
}
if (MI.getOpcode() == ARM::MVE_VMOV_q_rr) {
assert(MI.getOperand(4).isImm() && MI.getOperand(5).isImm());
if ((MI.getOperand(4).getImm() != 2 && MI.getOperand(4).getImm() != 3) ||
MI.getOperand(4).getImm() != MI.getOperand(5).getImm() + 2) {
ErrInfo = "Incorrect array index for MVE_VMOV_q_rr";
return false;
}
}
return true;
}

Expand Down
35 changes: 35 additions & 0 deletions llvm/lib/Target/ARM/ARMInstrMVE.td
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5845,6 +5845,41 @@ def MVE_VMOV_rr_q : MVE_VMOV_64bit<(outs rGPR:$Rt, rGPR:$Rt2), (ins MQPR:$Qd),
let AsmMatchConverter = "cvtMVEVMOVQtoDReg";
}

let Predicates = [HasMVEInt] in {
// Double lane moves. There are a number of patterns here. We know that the
// insertelt's will be in descending order by index, and need to match the 5
// patterns that might contain 2-0 or 3-1 pairs. These are:
// 3 2 1 0 -> vmovqrr 31; vmovqrr 20
// 3 2 1 -> vmovqrr 31; vmov 2
// 3 1 -> vmovqrr 31
// 2 1 0 -> vmovqrr 20; vmov 1
// 2 0 -> vmovqrr 20
// The other potential patterns will be handled by single lane inserts.
def : Pat<(insertelt (insertelt (insertelt (insertelt (v4i32 MQPR:$src1),
rGPR:$srcA, (i32 0)),
rGPR:$srcB, (i32 1)),
rGPR:$srcC, (i32 2)),
rGPR:$srcD, (i32 3)),
(MVE_VMOV_q_rr (MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcC, (i32 2), (i32 0)),
rGPR:$srcB, rGPR:$srcD, (i32 3), (i32 1))>;
def : Pat<(insertelt (insertelt (insertelt (v4i32 MQPR:$src1),
rGPR:$srcB, (i32 1)),
rGPR:$srcC, (i32 2)),
rGPR:$srcD, (i32 3)),
(MVE_VMOV_q_rr (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$srcC, (i32 2)),
rGPR:$srcB, rGPR:$srcD, (i32 3), (i32 1))>;
def : Pat<(insertelt (insertelt (v4i32 MQPR:$src1), rGPR:$srcA, (i32 1)), rGPR:$srcB, (i32 3)),
(MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcB, (i32 3), (i32 1))>;
def : Pat<(insertelt (insertelt (insertelt (v4i32 MQPR:$src1),
rGPR:$srcB, (i32 0)),
rGPR:$srcC, (i32 1)),
rGPR:$srcD, (i32 2)),
(MVE_VMOV_q_rr (MVE_VMOV_to_lane_32 MQPR:$src1, rGPR:$srcC, (i32 1)),
rGPR:$srcB, rGPR:$srcD, (i32 2), (i32 0))>;
def : Pat<(insertelt (insertelt (v4i32 MQPR:$src1), rGPR:$srcA, (i32 0)), rGPR:$srcB, (i32 2)),
(MVE_VMOV_q_rr MQPR:$src1, rGPR:$srcA, rGPR:$srcB, (i32 2), (i32 0))>;
}

// end of coproc mov

// start of MVE interleaving load/store
Expand Down
137 changes: 63 additions & 74 deletions llvm/test/CodeGen/Thumb2/active_lane_mask.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,28 +39,24 @@ define <7 x i32> @v7i32(i32 %index, i32 %TC, <7 x i32> %V1, <7 x i32> %V2) {
; CHECK-NEXT: adr r3, .LCPI1_0
; CHECK-NEXT: vdup.32 q1, r1
; CHECK-NEXT: vldrw.u32 q0, [r3]
; CHECK-NEXT: ldr r3, [sp, #32]
; CHECK-NEXT: vadd.i32 q2, q0, r1
; CHECK-NEXT: vdup.32 q0, r2
; CHECK-NEXT: vcmp.u32 hi, q1, q2
; CHECK-NEXT: ldr r2, [sp, #32]
; CHECK-NEXT: ldr r2, [sp, #40]
; CHECK-NEXT: vpnot
; CHECK-NEXT: vpst
; CHECK-NEXT: vcmpt.u32 hi, q0, q2
; CHECK-NEXT: vmov.32 q2[0], r2
; CHECK-NEXT: ldr r2, [sp, #36]
; CHECK-NEXT: vmov.32 q2[1], r2
; CHECK-NEXT: ldr r2, [sp, #40]
; CHECK-NEXT: vmov.32 q2[2], r2
; CHECK-NEXT: vmov q2[2], q2[0], r3, r2
; CHECK-NEXT: ldr r2, [sp, #44]
; CHECK-NEXT: vmov.32 q2[3], r2
; CHECK-NEXT: ldr r2, [sp]
; CHECK-NEXT: vmov.32 q3[0], r2
; CHECK-NEXT: ldr r2, [sp, #4]
; CHECK-NEXT: vmov.32 q3[1], r2
; CHECK-NEXT: ldr r3, [sp, #36]
; CHECK-NEXT: vmov q2[3], q2[1], r3, r2
; CHECK-NEXT: ldr r2, [sp, #8]
; CHECK-NEXT: vmov.32 q3[2], r2
; CHECK-NEXT: ldr r3, [sp]
; CHECK-NEXT: vmov q3[2], q3[0], r3, r2
; CHECK-NEXT: ldr r2, [sp, #12]
; CHECK-NEXT: vmov.32 q3[3], r2
; CHECK-NEXT: ldr r3, [sp, #4]
; CHECK-NEXT: vmov q3[3], q3[1], r3, r2
; CHECK-NEXT: adr r2, .LCPI1_1
; CHECK-NEXT: vpsel q2, q3, q2
; CHECK-NEXT: vstrw.32 q2, [r0]
Expand All @@ -70,21 +66,19 @@ define <7 x i32> @v7i32(i32 %index, i32 %TC, <7 x i32> %V1, <7 x i32> %V2) {
; CHECK-NEXT: vcmp.u32 hi, q1, q2
; CHECK-NEXT: vmrs r1, p0
; CHECK-NEXT: eors r1, r2
; CHECK-NEXT: ldr r2, [sp, #48]
; CHECK-NEXT: vmsr p0, r1
; CHECK-NEXT: ldr r1, [sp, #48]
; CHECK-NEXT: ldr r1, [sp, #52]
; CHECK-NEXT: vpst
; CHECK-NEXT: vcmpt.u32 hi, q0, q2
; CHECK-NEXT: vmov.32 q0[0], r1
; CHECK-NEXT: ldr r1, [sp, #52]
; CHECK-NEXT: vmov.32 q0[1], r1
; CHECK-NEXT: ldr r1, [sp, #56]
; CHECK-NEXT: vmov.32 q0[2], r1
; CHECK-NEXT: ldr r1, [sp, #16]
; CHECK-NEXT: vmov.32 q1[0], r1
; CHECK-NEXT: vmov q0[2], q0[0], r2, r1
; CHECK-NEXT: ldr r1, [sp, #20]
; CHECK-NEXT: ldr r2, [sp, #16]
; CHECK-NEXT: vmov.32 q1[1], r1
; CHECK-NEXT: ldr r1, [sp, #24]
; CHECK-NEXT: vmov.32 q1[2], r1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r1
; CHECK-NEXT: vpsel q0, q1, q0
; CHECK-NEXT: vmov r1, s2
; CHECK-NEXT: vmov.f32 s2, s1
Expand Down Expand Up @@ -413,81 +407,75 @@ define <16 x i8> @v16i8(i32 %index, i32 %TC, <16 x i8> %V1, <16 x i8> %V2) {
define void @test_width2(i32* nocapture readnone %x, i32* nocapture %y, i8 zeroext %m) {
; CHECK-LABEL: test_width2:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: push {r4, r5, r6, lr}
; CHECK-NEXT: vpush {d8, d9, d10, d11}
; CHECK-NEXT: push.w {r4, r5, r6, r7, r8, r9, lr}
; CHECK-NEXT: sub sp, #4
; CHECK-NEXT: vpush {d8, d9}
; CHECK-NEXT: sub sp, #8
; CHECK-NEXT: cmp r2, #0
; CHECK-NEXT: beq.w .LBB4_3
; CHECK-NEXT: @ %bb.1: @ %for.body.preheader
; CHECK-NEXT: adds r0, r2, #1
; CHECK-NEXT: movs r3, #1
; CHECK-NEXT: vmov q1[2], q1[0], r2, r2
; CHECK-NEXT: bic r0, r0, #1
; CHECK-NEXT: vmov.32 q2[0], r2
; CHECK-NEXT: adr r2, .LCPI4_0
; CHECK-NEXT: subs r0, #2
; CHECK-NEXT: movs r3, #1
; CHECK-NEXT: vmov.i64 q0, #0xffffffff
; CHECK-NEXT: vmov.32 q2[2], r2
; CHECK-NEXT: movs r6, #0
; CHECK-NEXT: vldrw.u32 q2, [r2]
; CHECK-NEXT: add.w lr, r3, r0, lsr #1
; CHECK-NEXT: adr r3, .LCPI4_0
; CHECK-NEXT: mov.w r8, #0
; CHECK-NEXT: dls lr, lr
; CHECK-NEXT: vldrw.u32 q1, [r3]
; CHECK-NEXT: vand q2, q2, q0
; CHECK-NEXT: vand q1, q1, q0
; CHECK-NEXT: .LBB4_2: @ %vector.body
; CHECK-NEXT: @ =>This Inner Loop Header: Depth=1
; CHECK-NEXT: vmov.32 q3[0], r6
; CHECK-NEXT: vmov r5, s8
; CHECK-NEXT: vmov.32 q3[2], r6
; CHECK-NEXT: vmov r0, s9
; CHECK-NEXT: vmov q3[2], q3[0], r8, r8
; CHECK-NEXT: vmov r7, s6
; CHECK-NEXT: vand q3, q3, q0
; CHECK-NEXT: adds r6, #2
; CHECK-NEXT: vmov r6, s7
; CHECK-NEXT: vmov r3, s14
; CHECK-NEXT: add.w r8, r8, #2
; CHECK-NEXT: vmov r9, s12
; CHECK-NEXT: vmov r2, s15
; CHECK-NEXT: vmov r0, s5
; CHECK-NEXT: adds r3, #1
; CHECK-NEXT: adc r12, r2, #0
; CHECK-NEXT: vmov r2, s12
; CHECK-NEXT: vmov.32 q3[0], r2
; CHECK-NEXT: vmov.32 q3[2], r3
; CHECK-NEXT: vmov q3[2], q3[0], r9, r3
; CHECK-NEXT: vand q3, q3, q0
; CHECK-NEXT: vmov r4, s12
; CHECK-NEXT: teq.w r4, r2
; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: tst.w r2, #1
; CHECK-NEXT: csetm r2, ne
; CHECK-NEXT: vmov.32 q4[0], r2
; CHECK-NEXT: vmov.32 q4[1], r2
; CHECK-NEXT: vmov r2, s14
; CHECK-NEXT: eors r3, r2
; CHECK-NEXT: orrs.w r3, r3, r12
; CHECK-NEXT: cset r3, ne
; CHECK-NEXT: tst.w r3, #1
; CHECK-NEXT: csetm r3, ne
; CHECK-NEXT: subs r5, r4, r5
; CHECK-NEXT: vmov.32 q4[2], r3
; CHECK-NEXT: vmov r5, s10
; CHECK-NEXT: vmov.32 q4[3], r3
; CHECK-NEXT: vmov r3, s13
; CHECK-NEXT: veor q4, q4, q1
; CHECK-NEXT: sbcs.w r0, r3, r0
; CHECK-NEXT: vmov r3, s11
; CHECK-NEXT: mov.w r0, #0
; CHECK-NEXT: adc r12, r2, #0
; CHECK-NEXT: vmov r5, s14
; CHECK-NEXT: vmov r4, s15
; CHECK-NEXT: vmov r2, s4
; CHECK-NEXT: subs r7, r5, r7
; CHECK-NEXT: vmov r7, s12
; CHECK-NEXT: sbcs r4, r6
; CHECK-NEXT: vmov r6, s13
; CHECK-NEXT: mov.w r4, #0
; CHECK-NEXT: it lo
; CHECK-NEXT: movlo r0, #1
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, ne
; CHECK-NEXT: subs r2, r2, r5
; CHECK-NEXT: vmov.32 q5[0], r0
; CHECK-NEXT: vmov.32 q5[1], r0
; CHECK-NEXT: vmov r0, s15
; CHECK-NEXT: @ implicit-def: $q3
; CHECK-NEXT: sbcs r0, r3
; CHECK-NEXT: movlo r4, #1
; CHECK-NEXT: cmp r4, #0
; CHECK-NEXT: csetm r4, ne
; CHECK-NEXT: subs r2, r7, r2
; CHECK-NEXT: sbcs.w r0, r6, r0
; CHECK-NEXT: mov.w r0, #0
; CHECK-NEXT: it lo
; CHECK-NEXT: movlo r0, #1
; CHECK-NEXT: cmp r0, #0
; CHECK-NEXT: csetm r0, ne
; CHECK-NEXT: vmov.32 q5[2], r0
; CHECK-NEXT: vmov.32 q5[3], r0
; CHECK-NEXT: vand q4, q4, q5
; CHECK-NEXT: vmov q3[2], q3[0], r0, r4
; CHECK-NEXT: vmov q3[3], q3[1], r0, r4
; CHECK-NEXT: eor.w r0, r5, r3
; CHECK-NEXT: orrs.w r0, r0, r12
; CHECK-NEXT: cset r0, ne
; CHECK-NEXT: tst.w r0, #1
; CHECK-NEXT: csetm r0, ne
; CHECK-NEXT: teq.w r7, r9
; CHECK-NEXT: cset r2, ne
; CHECK-NEXT: tst.w r2, #1
; CHECK-NEXT: csetm r2, ne
; CHECK-NEXT: vmov q4[2], q4[0], r2, r0
; CHECK-NEXT: vmov q4[3], q4[1], r2, r0
; CHECK-NEXT: veor q4, q4, q2
; CHECK-NEXT: vand q4, q4, q3
; CHECK-NEXT: @ implicit-def: $q3
; CHECK-NEXT: vmov r2, s16
; CHECK-NEXT: vmov r0, s18
; CHECK-NEXT: and r2, r2, #1
Expand Down Expand Up @@ -519,8 +507,9 @@ define void @test_width2(i32* nocapture readnone %x, i32* nocapture %y, i8 zeroe
; CHECK-NEXT: le lr, .LBB4_2
; CHECK-NEXT: .LBB4_3: @ %for.cond.cleanup
; CHECK-NEXT: add sp, #8
; CHECK-NEXT: vpop {d8, d9, d10, d11}
; CHECK-NEXT: pop {r4, r5, r6, pc}
; CHECK-NEXT: vpop {d8, d9}
; CHECK-NEXT: add sp, #4
; CHECK-NEXT: pop.w {r4, r5, r6, r7, r8, r9, pc}
; CHECK-NEXT: .p2align 4
; CHECK-NEXT: @ %bb.4:
; CHECK-NEXT: .LCPI4_0:
Expand Down
23 changes: 10 additions & 13 deletions llvm/test/CodeGen/Thumb2/mve-abs.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,23 +40,20 @@ entry:
define arm_aapcs_vfpcc <2 x i64> @abs_v2i64(<2 x i64> %s1) {
; CHECK-LABEL: abs_v2i64:
; CHECK: @ %bb.0: @ %entry
; CHECK-NEXT: vmov r0, s1
; CHECK-NEXT: vmov r1, s0
; CHECK-NEXT: adds.w r1, r1, r0, asr #31
; CHECK-NEXT: adc.w r2, r0, r0, asr #31
; CHECK-NEXT: eor.w r2, r2, r0, asr #31
; CHECK-NEXT: eor.w r0, r1, r0, asr #31
; CHECK-NEXT: vmov.32 q1[0], r0
; CHECK-NEXT: vmov r0, s3
; CHECK-NEXT: vmov r1, s2
; CHECK-NEXT: vmov.32 q1[1], r2
; CHECK-NEXT: vmov r3, s1
; CHECK-NEXT: vmov r2, s0
; CHECK-NEXT: adds.w r1, r1, r0, asr #31
; CHECK-NEXT: adc.w r12, r0, r0, asr #31
; CHECK-NEXT: eor.w r1, r1, r0, asr #31
; CHECK-NEXT: vmov.32 q1[2], r1
; CHECK-NEXT: adc.w r1, r0, r0, asr #31
; CHECK-NEXT: eor.w r0, r1, r0, asr #31
; CHECK-NEXT: vmov.32 q1[3], r0
; CHECK-NEXT: vmov q0, q1
; CHECK-NEXT: adds.w r2, r2, r3, asr #31
; CHECK-NEXT: eor.w r0, r12, r0, asr #31
; CHECK-NEXT: eor.w r2, r2, r3, asr #31
; CHECK-NEXT: vmov q0[2], q0[0], r2, r1
; CHECK-NEXT: adc.w r1, r3, r3, asr #31
; CHECK-NEXT: eor.w r1, r1, r3, asr #31
; CHECK-NEXT: vmov q0[3], q0[1], r1, r0
; CHECK-NEXT: bx lr
entry:
%0 = icmp slt <2 x i64> %s1, zeroinitializer
Expand Down
Loading

0 comments on commit e1c1adf

Please sign in to comment.