[RISCV] Support __builtin_nontemporal_load/store by MachineMemOperand

Differential Revision: https://reviews.llvm.org/D143361

llvm/lib/Target/RISCV/CMakeLists.txt

@@ -26,6 +26,7 @@ add_llvm_target(RISCVCodeGen
   RISCVExpandPseudoInsts.cpp
   RISCVFrameLowering.cpp
   RISCVGatherScatterLowering.cpp
+  RISCVInsertNTLHInsts.cpp
   RISCVInsertVSETVLI.cpp
   RISCVInstrInfo.cpp
   RISCVISelDAGToDAG.cpp

llvm/lib/Target/RISCV/RISCV.h

@@ -62,6 +62,9 @@ void initializeRISCVPreRAExpandPseudoPass(PassRegistry &);
 FunctionPass *createRISCVExpandAtomicPseudoPass();
 void initializeRISCVExpandAtomicPseudoPass(PassRegistry &);
 
+FunctionPass *createRISCVInsertNTLHInstsPass();
+void initializeRISCVInsertNTLHInstsPass(PassRegistry &);
+
 FunctionPass *createRISCVInsertVSETVLIPass();
 void initializeRISCVInsertVSETVLIPass(PassRegistry &);
 

llvm/lib/Target/RISCV/RISCVInsertNTLHInsts.cpp

@@ -0,0 +1,92 @@
+//===-- RISCVInsertNTLHInsts.cpp - Insert NTLH extension instrution -------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements a function pass that inserts non-temporal hint
+// instructions where needed.
+//
+// It checks the MachineMemOperand of all MachineInstr.
+// If the instruction has a MachineMemOperand and isNontemporal is true,
+// then ntlh instruction is inserted before it.
+//
+//===----------------------------------------------------------------------===//
+
+#include "RISCV.h"
+#include "RISCVInstrInfo.h"
+#include "RISCVTargetMachine.h"
+
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+
+using namespace llvm;
+
+#define RISCV_INSERT_NTLH_INSTS_NAME "RISC-V insert NTLH instruction pass"
+
+namespace {
+
+class RISCVInsertNTLHInsts : public MachineFunctionPass {
+public:
+  const RISCVInstrInfo *TII;
+  static char ID;
+
+  RISCVInsertNTLHInsts() : MachineFunctionPass(ID) {
+    initializeRISCVInsertNTLHInstsPass(*PassRegistry::getPassRegistry());
+  }
+
+  bool runOnMachineFunction(MachineFunction &MF) override;
+
+  void getAnalysisUsage(AnalysisUsage &AU) const override {
+    AU.setPreservesCFG();
+    MachineFunctionPass::getAnalysisUsage(AU);
+  }
+
+  StringRef getPassName() const override {
+    return RISCV_INSERT_NTLH_INSTS_NAME;
+  }
+};
+
+} // end of anonymous namespace
+
+char RISCVInsertNTLHInsts::ID = 0;
+
+bool RISCVInsertNTLHInsts::runOnMachineFunction(MachineFunction &MF) {
+  const auto &ST = MF.getSubtarget<RISCVSubtarget>();
+  TII = ST.getInstrInfo();
+
+  if (!ST.hasStdExtZihintntl())
+    return false;
+
+  bool Changed = false;
+  for (auto &MBB : MF) {
+    for (auto &MBBI : MBB) {
+      if (MBBI.memoperands_empty())
+        continue;
+      MachineMemOperand *MMO = *(MBBI.memoperands_begin());
+      if (MMO->isNonTemporal()) {
+        DebugLoc DL = MBBI.getDebugLoc();
+        if (ST.hasStdExtCOrZca() && ST.enableRVCHintInstrs())
+          BuildMI(MBB, MBBI, DL, TII->get(RISCV::PseudoCNTLALL));
+        else
+          BuildMI(MBB, MBBI, DL, TII->get(RISCV::PseudoNTLALL));
+        Changed = true;
+      }
+    }
+  }
+
+  return Changed;
+}
+
+INITIALIZE_PASS(RISCVInsertNTLHInsts, "riscv-insert-ntlh-insts",
+                RISCV_INSERT_NTLH_INSTS_NAME, false, false)
+
+namespace llvm {
+
+FunctionPass *createRISCVInsertNTLHInstsPass() {
+  return new RISCVInsertNTLHInsts();
+}
+
+} // end of namespace llvm

llvm/lib/Target/RISCV/RISCVInstrInfo.cpp

@@ -1243,6 +1243,20 @@ unsigned RISCVInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const {
                               *TM.getMCAsmInfo());
   }
 
+  if (!MI.memoperands_empty()) {
+    MachineMemOperand *MMO = *(MI.memoperands_begin());
+    const MachineFunction &MF = *MI.getParent()->getParent();
+    const auto &ST = MF.getSubtarget<RISCVSubtarget>();
+    if (ST.hasStdExtZihintntl() && MMO->isNonTemporal()) {
+      if (ST.hasStdExtCOrZca() && ST.enableRVCHintInstrs()) {
+        if (isCompressibleInst(MI, STI))
+          return 4; // c.ntl.all + c.load/c.store
+        return 6;   // c.ntl.all + load/store
+      }
+      return 8; // ntl.all + load/store
+    }
+  }
+
   if (MI.getParent() && MI.getParent()->getParent()) {
     if (isCompressibleInst(MI, STI))
       return 2;

llvm/lib/Target/RISCV/RISCVInstrInfo.td

@@ -1920,3 +1920,4 @@ include "RISCVInstrInfoZicond.td"
 
 include "RISCVInstrInfoXVentana.td"
 include "RISCVInstrInfoXTHead.td"
+include "RISCVInstrInfoZihintntl.td"

llvm/lib/Target/RISCV/RISCVInstrInfoZihintntl.td

@@ -0,0 +1,22 @@
+//===RISCVInstrInfoZihintntl.td - 'Zihintntl' instructions -*- tablegen -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// This file describes the RISC-V instructions from Non-Temporal Locality 
+/// Hints extension document (zihintntl).
+///
+//===----------------------------------------------------------------------===//
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 4 in {
+  def PseudoNTLALL :  Pseudo<(outs), (ins), [], "ntl.all">, 
+                             PseudoInstExpansion<(ADD X0, X0, X5)>;
+}
+
+let hasSideEffects = 0, mayLoad = 0, mayStore = 0, Size = 2 in {
+  def PseudoCNTLALL :  Pseudo<(outs), (ins), [], "c.ntl.all">,
+                              PseudoInstExpansion<(C_ADD_HINT X0, X0, X5)>;
+}

llvm/lib/Target/RISCV/RISCVTargetMachine.cpp

@@ -83,6 +83,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
   initializeRISCVOptWInstrsPass(*PR);
   initializeRISCVPreRAExpandPseudoPass(*PR);
   initializeRISCVExpandPseudoPass(*PR);
+  initializeRISCVInsertNTLHInstsPass(*PR);
   initializeRISCVInsertVSETVLIPass(*PR);
   initializeRISCVDAGToDAGISelPass(*PR);
   initializeRISCVInitUndefPass(*PR);
@@ -348,6 +349,7 @@ void RISCVPassConfig::addPreEmitPass() {
 
 void RISCVPassConfig::addPreEmitPass2() {
   addPass(createRISCVExpandPseudoPass());
+  addPass(createRISCVInsertNTLHInstsPass());
 
   // Schedule the expansion of AMOs at the last possible moment, avoiding the
   // possibility for other passes to break the requirements for forward

llvm/test/CodeGen/RISCV/O0-pipeline.ll

@@ -64,6 +64,7 @@
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Stack Frame Layout Analysis
 ; CHECK-NEXT:       RISC-V pseudo instruction expansion pass
+; CHECK-NEXT:       RISC-V insert NTLH instruction pass
 ; CHECK-NEXT:       RISC-V atomic pseudo instruction expansion pass
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter

llvm/test/CodeGen/RISCV/O3-pipeline.ll

@@ -177,6 +177,7 @@
 ; CHECK-NEXT:       Machine Optimization Remark Emitter
 ; CHECK-NEXT:       Stack Frame Layout Analysis
 ; CHECK-NEXT:       RISC-V pseudo instruction expansion pass
+; CHECK-NEXT:       RISC-V insert NTLH instruction pass
 ; CHECK-NEXT:       RISC-V atomic pseudo instruction expansion pass
 ; CHECK-NEXT:       Lazy Machine Block Frequency Analysis
 ; CHECK-NEXT:       Machine Optimization Remark Emitter

llvm/test/CodeGen/RISCV/nontemporal-scalable.ll

@@ -0,0 +1,133 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
+; RUN: llc -mtriple=riscv64 -mattr=+experimental-zihintntl,+f,+d,+zfh,+v < %s | FileCheck %s -check-prefix=CHECK-RV64V
+; RUN: llc -mtriple=riscv32 -mattr=+experimental-zihintntl,+f,+d,+zfh,+v < %s | FileCheck %s -check-prefix=CHECK-RV32V
+
+define <vscale x 2 x i64> @test_nontemporal_load_nxv2i64(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_load_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_load_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2re64.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 2 x i64>, ptr %p, !nontemporal !0
+  ret <vscale x 2 x i64> %1
+}
+
+define <vscale x 4 x i32> @test_nontemporal_load_nxv4i32(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_load_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_load_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2re32.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 4 x i32>, ptr %p, !nontemporal !0
+  ret <vscale x 4 x i32> %1
+}
+
+define <vscale x 8 x i16> @test_nontemporal_load_nxv8i16(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_load_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_load_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2re16.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 8 x i16>, ptr %p, !nontemporal !0
+  ret <vscale x 8 x i16> %1
+}
+
+define <vscale x 16 x i8> @test_nontemporal_load_nxv16i8(ptr %p) {
+; CHECK-RV64V-LABEL: test_nontemporal_load_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_load_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vl2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  %1 = load <vscale x 16 x i8>, ptr %p, !nontemporal !0
+  ret <vscale x 16 x i8> %1
+}
+
+define void @test_nontemporal_store_nxv2i64(ptr %p, <vscale x 2 x i64> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_store_nxv2i64:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_store_nxv2i64:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 2 x i64> %v, ptr %p, !nontemporal !0
+  ret void
+}
+
+define void @test_nontemporal_store_nxv4i32(ptr %p, <vscale x 4 x i32> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_store_nxv4i32:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_store_nxv4i32:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 4 x i32> %v, ptr %p, !nontemporal !0
+  ret void
+}
+
+define void @test_nontemporal_store_nxv8i16(ptr %p, <vscale x 8 x i16> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_store_nxv8i16:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_store_nxv8i16:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 8 x i16> %v, ptr %p, !nontemporal !0
+  ret void
+}
+
+define void @test_nontemporal_store_nxv16i8(ptr %p, <vscale x 16 x i8> %v) {
+; CHECK-RV64V-LABEL: test_nontemporal_store_nxv16i8:
+; CHECK-RV64V:       # %bb.0:
+; CHECK-RV64V-NEXT:    ntl.all
+; CHECK-RV64V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV64V-NEXT:    ret
+;
+; CHECK-RV32V-LABEL: test_nontemporal_store_nxv16i8:
+; CHECK-RV32V:       # %bb.0:
+; CHECK-RV32V-NEXT:    ntl.all
+; CHECK-RV32V-NEXT:    vs2r.v v8, (a0)
+; CHECK-RV32V-NEXT:    ret
+  store <vscale x 16 x i8> %v, ptr %p, !nontemporal !0
+  ret void
+}
+
+!0 = !{i32 1}