diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp index 80340ee815094f..3ff6024b50ca04 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp @@ -287,9 +287,10 @@ bool RISCVExpandPseudo::expandVSPILL(MachineBasicBlock &MBB, const TargetRegisterInfo *TRI = MBB.getParent()->getSubtarget().getRegisterInfo(); DebugLoc DL = MBBI->getDebugLoc(); - Register SrcReg = MBBI->getOperand(0).getReg(); - Register Base = MBBI->getOperand(1).getReg(); - Register VL = MBBI->getOperand(2).getReg(); + Register AddrInc = MBBI->getOperand(0).getReg(); + Register SrcReg = MBBI->getOperand(1).getReg(); + Register Base = MBBI->getOperand(2).getReg(); + Register VL = MBBI->getOperand(3).getReg(); auto ZvlssegInfo = TII->isRVVSpillForZvlsseg(MBBI->getOpcode()); if (!ZvlssegInfo) return false; @@ -318,10 +319,12 @@ bool RISCVExpandPseudo::expandVSPILL(MachineBasicBlock &MBB, .addReg(TRI->getSubReg(SrcReg, SubRegIdx + I)) .addReg(Base) .addMemOperand(*(MBBI->memoperands_begin())); - if (I != NF - 1) - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADD), Base) + if (I != NF - 1) { + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADD), AddrInc) .addReg(Base) .addReg(VL); + Base = AddrInc; + } } MBBI->eraseFromParent(); return true; @@ -333,8 +336,9 @@ bool RISCVExpandPseudo::expandVRELOAD(MachineBasicBlock &MBB, MBB.getParent()->getSubtarget().getRegisterInfo(); DebugLoc DL = MBBI->getDebugLoc(); Register DestReg = MBBI->getOperand(0).getReg(); - Register Base = MBBI->getOperand(1).getReg(); - Register VL = MBBI->getOperand(2).getReg(); + Register AddrInc = MBBI->getOperand(1).getReg(); + Register Base = MBBI->getOperand(2).getReg(); + Register VL = MBBI->getOperand(3).getReg(); auto ZvlssegInfo = TII->isRVVSpillForZvlsseg(MBBI->getOpcode()); if (!ZvlssegInfo) return false; @@ -363,10 +367,12 @@ bool RISCVExpandPseudo::expandVRELOAD(MachineBasicBlock &MBB, TRI->getSubReg(DestReg, SubRegIdx + I)) .addReg(Base) .addMemOperand(*(MBBI->memoperands_begin())); - if (I != NF - 1) - BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADD), Base) + if (I != NF - 1) { + BuildMI(MBB, MBBI, DL, TII->get(RISCV::ADD), AddrInc) .addReg(Base) .addReg(VL); + Base = AddrInc; + } } MBBI->eraseFromParent(); return true; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp index fcfd98ec7ed93d..c216662a604f1c 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp +++ b/llvm/lib/Target/RISCV/RISCVInstrInfo.cpp @@ -311,10 +311,17 @@ void RISCVInstrInfo::storeRegToStackSlot(MachineBasicBlock &MBB, MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); MFI.setStackID(FI, TargetStackID::ScalableVector); - auto MIB = BuildMI(MBB, I, DL, get(Opcode)) - .addReg(SrcReg, getKillRegState(IsKill)) - .addFrameIndex(FI) - .addMemOperand(MMO); + auto MIB = BuildMI(MBB, I, DL, get(Opcode)); + if (IsZvlsseg) { + // We need a GPR register to hold the incremented address for each subreg + // after expansion. + Register AddrInc = + MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); + MIB.addReg(AddrInc, RegState::Define); + } + MIB.addReg(SrcReg, getKillRegState(IsKill)) + .addFrameIndex(FI) + .addMemOperand(MMO); if (IsZvlsseg) { // For spilling/reloading Zvlsseg registers, append the dummy field for // the scaled vector length. The argument will be used when expanding @@ -405,9 +412,15 @@ void RISCVInstrInfo::loadRegFromStackSlot(MachineBasicBlock &MBB, MemoryLocation::UnknownSize, MFI.getObjectAlign(FI)); MFI.setStackID(FI, TargetStackID::ScalableVector); - auto MIB = BuildMI(MBB, I, DL, get(Opcode), DstReg) - .addFrameIndex(FI) - .addMemOperand(MMO); + auto MIB = BuildMI(MBB, I, DL, get(Opcode), DstReg); + if (IsZvlsseg) { + // We need a GPR register to hold the incremented address for each subreg + // after expansion. + Register AddrInc = + MF->getRegInfo().createVirtualRegister(&RISCV::GPRRegClass); + MIB.addReg(AddrInc, RegState::Define); + } + MIB.addFrameIndex(FI).addMemOperand(MMO); if (IsZvlsseg) { // For spilling/reloading Zvlsseg registers, append the dummy field for // the scaled vector length. The argument will be used when expanding diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index e8b430031c1961..1ebc0824d65d58 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -3356,11 +3356,11 @@ foreach lmul = MxList.m in { defvar vreg = SegRegClass.RC; let hasSideEffects = 0, mayLoad = 0, mayStore = 1, isCodeGenOnly = 1 in { def "PseudoVSPILL" # nf # "_" # lmul.MX : - Pseudo<(outs), (ins vreg:$rs1, GPR:$rs2, GPR:$vlenb), []>; + Pseudo<(outs GPR:$addrinc), (ins vreg:$rs1, GPR:$rs2, GPR:$vlenb), []>; } let hasSideEffects = 0, mayLoad = 1, mayStore = 0, isCodeGenOnly = 1 in { def "PseudoVRELOAD" # nf # "_" # lmul.MX : - Pseudo<(outs vreg:$rs1), (ins GPR:$rs2, GPR:$vlenb), []>; + Pseudo<(outs vreg:$rs1, GPR:$addrinc), (ins GPR:$rs2, GPR:$vlenb), []>; } } } diff --git a/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir b/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir index c8ec11341d673e..cbde2565348eb9 100644 --- a/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir +++ b/llvm/test/CodeGen/RISCV/rvv/zvlsseg-spill.mir @@ -29,10 +29,10 @@ body: | ; CHECK: $v0_v1_v2_v3_v4_v5_v6 = PseudoVLSEG7E64_V_M1 renamable $x10, $noreg, 6, implicit $vl, implicit $vtype ; CHECK: $x11 = ADDI $x2, 16 ; CHECK: $x12 = PseudoReadVLENB - ; CHECK: PseudoVSPILL7_M1 killed renamable $v0_v1_v2_v3_v4_v5_v6, killed $x11, killed $x12 + ; CHECK: dead renamable $x11 = PseudoVSPILL7_M1 killed renamable $v0_v1_v2_v3_v4_v5_v6, killed $x11, killed $x12 ; CHECK: $x11 = ADDI $x2, 16 ; CHECK: $x12 = PseudoReadVLENB - ; CHECK: dead renamable $v7_v8_v9_v10_v11_v12_v13 = PseudoVRELOAD7_M1 killed $x11, killed $x12, implicit-def $v8 + ; CHECK: dead renamable $v7_v8_v9_v10_v11_v12_v13, dead renamable $x11 = PseudoVRELOAD7_M1 killed $x11, killed $x12, implicit-def $v8 ; CHECK: VS1R_V killed $v8, killed renamable $x10 ; CHECK: $x10 = frame-destroy PseudoReadVLENB ; CHECK: $x10 = frame-destroy SLLI killed $x10, 3 @@ -42,8 +42,8 @@ body: | %0:gpr = COPY $x10 %1:gprnox0 = COPY $x11 $v0_v1_v2_v3_v4_v5_v6 = PseudoVLSEG7E64_V_M1 %0, %1, 6 - PseudoVSPILL7_M1 killed renamable $v0_v1_v2_v3_v4_v5_v6, %stack.0, $x0 - renamable $v7_v8_v9_v10_v11_v12_v13 = PseudoVRELOAD7_M1 %stack.0, $x0 + %2:gpr = PseudoVSPILL7_M1 killed renamable $v0_v1_v2_v3_v4_v5_v6, %stack.0, $x0 + renamable $v7_v8_v9_v10_v11_v12_v13, %3:gpr = PseudoVRELOAD7_M1 %stack.0, $x0 VS1R_V killed $v8, %0:gpr PseudoRET ...