Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions llvm/include/llvm/IR/Intrinsics.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ class LLVMContext;
class Module;
class AttributeList;
class AttributeSet;
class raw_ostream;
class Constant;

/// This namespace contains an enum with a value for every intrinsic/builtin
/// function known by LLVM. The enum values are returned by
Expand Down Expand Up @@ -81,6 +83,9 @@ namespace Intrinsic {
/// Returns true if the intrinsic can be overloaded.
LLVM_ABI bool isOverloaded(ID id);

/// Returns true if the intrinsic has pretty printed immediate arguments.
LLVM_ABI bool hasPrettyPrintedArgs(ID id);

/// isTargetIntrinsic - Returns true if IID is an intrinsic specific to a
/// certain target. If it is a generic intrinsic false is returned.
LLVM_ABI bool isTargetIntrinsic(ID IID);
Expand Down Expand Up @@ -284,6 +289,10 @@ namespace Intrinsic {
/// N.
LLVM_ABI Intrinsic::ID getDeinterleaveIntrinsicID(unsigned Factor);

/// Print the argument info for the arguments with ArgInfo.
LLVM_ABI void printImmArg(ID IID, unsigned ArgIdx, raw_ostream &OS,
const Constant *ImmArgVal);

} // namespace Intrinsic

} // namespace llvm
Expand Down
8 changes: 8 additions & 0 deletions llvm/include/llvm/IR/Intrinsics.td
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,14 @@ class NoUndef<AttrIndex idx> : IntrinsicProperty {
int ArgNo = idx.Value;
}

// ArgInfo - The specified argument has an argument name and an optional argument printing
// function for diagnostic output.
class ArgInfo<AttrIndex idx, string argname, string funcname = ""> : IntrinsicProperty {
int ArgNo = idx.Value;
string ArgName = argname;
string FunctionName = funcname;
}

// NonNull - The return value or specified argument is not null.
class NonNull<AttrIndex idx> : IntrinsicProperty {
int ArgNo = idx.Value;
Expand Down
16 changes: 15 additions & 1 deletion llvm/include/llvm/IR/IntrinsicsNVVM.td
Original file line number Diff line number Diff line change
Expand Up @@ -2871,7 +2871,14 @@ foreach sp = [0, 1] in {
defvar nargs = !size(args);
defvar scale_d_imm = ArgIndex<!sub(nargs, 1)>;
defvar scale_d_imm_range = [ImmArg<scale_d_imm>, Range<scale_d_imm, 0, 16>];
defvar intrinsic_properties = !listconcat(

// Check if this is the specific llvm.nvvm.tcgen05.mma.tensor intrinsic.
defvar is_target_intrinsic = !and(!eq(sp, 0),
!eq(space, "tensor"),
!eq(scale_d, 0),
!eq(ashift, 0));

defvar base_properties = !listconcat(
mma.common_intr_props,
!if(!eq(scale_d, 1), scale_d_imm_range, []),
[Range<ArgIndex<nargs>, 0, !if(!eq(scale_d, 1), 2, 4)>, // kind
Expand All @@ -2881,6 +2888,13 @@ foreach sp = [0, 1] in {
]
);

defvar intrinsic_properties = !if(is_target_intrinsic,
!listconcat(base_properties,
[ArgInfo<ArgIndex<nargs>, "kind", "printTcgen05MMAKind">,
ArgInfo<ArgIndex<!add(nargs, 1)>, "cta_group">,
ArgInfo<ArgIndex<!add(nargs, 2)>, "collector", "printTcgen05CollectorUsageOp">]),
base_properties);

def mma.record:
DefaultAttrsIntrinsicFlags<[], args, flags, intrinsic_properties,
mma.intr>;
Expand Down
48 changes: 48 additions & 0 deletions llvm/include/llvm/IR/NVVMIntrinsicUtils.h
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@
#include <stdint.h>

#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/APInt.h"
#include "llvm/IR/Constants.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/IntrinsicsNVPTX.h"
#include "llvm/Support/raw_ostream.h"

namespace llvm {
namespace nvvm {
Expand Down Expand Up @@ -659,6 +662,51 @@ inline APFloat::roundingMode GetFMARoundingMode(Intrinsic::ID IntrinsicID) {
llvm_unreachable("Invalid FP instrinsic rounding mode for NVVM fma");
}

inline void printTcgen05MMAKind(raw_ostream &OS, const Constant *ImmArgVal) {
if (const ConstantInt *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
uint64_t Val = CI->getZExtValue();
switch (static_cast<Tcgen05MMAKind>(Val)) {
case Tcgen05MMAKind::F16:
OS << "f16";
return;
case Tcgen05MMAKind::TF32:
OS << "tf32";
return;
case Tcgen05MMAKind::F8F6F4:
OS << "f8f6f4";
return;
case Tcgen05MMAKind::I8:
OS << "i8";
return;
}
}
llvm_unreachable(
"printTcgen05MMAKind called with invalid value for immediate argument");
}

inline void printTcgen05CollectorUsageOp(raw_ostream &OS,
const Constant *ImmArgVal) {
if (const ConstantInt *CI = dyn_cast<ConstantInt>(ImmArgVal)) {
uint64_t Val = CI->getZExtValue();
switch (static_cast<Tcgen05CollectorUsageOp>(Val)) {
case Tcgen05CollectorUsageOp::DISCARD:
OS << "discard";
return;
case Tcgen05CollectorUsageOp::LASTUSE:
OS << "lastuse";
return;
case Tcgen05CollectorUsageOp::FILL:
OS << "fill";
return;
case Tcgen05CollectorUsageOp::USE:
OS << "use";
return;
}
}
llvm_unreachable("printTcgen05CollectorUsageOp called with invalid value for "
"immediate argument");
}

} // namespace nvvm
} // namespace llvm
#endif // LLVM_IR_NVVMINTRINSICUTILS_H
52 changes: 42 additions & 10 deletions llvm/lib/IR/AsmWriter.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@
#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Metadata.h"
#include "llvm/IR/Module.h"
Expand Down Expand Up @@ -106,6 +107,10 @@ static cl::opt<bool> PreserveAssemblyUseListOrder(
"preserve-ll-uselistorder", cl::Hidden, cl::init(false),
cl::desc("Preserve use-list order when writing LLVM assembly."));

static cl::opt<bool> PrintFormattedIntrinsics(
"print-formatted-intrinsics",
cl::desc("Enable pretty print format for intrinsic arguments"));

// Make virtual table appear in this compilation unit.
AssemblyAnnotationWriter::~AssemblyAnnotationWriter() = default;

Expand Down Expand Up @@ -2841,6 +2846,7 @@ class AssemblyWriter {
SetVector<const Comdat *> Comdats;
bool IsForDebug;
bool ShouldPreserveUseListOrder;
bool PrettyPrintIntrinsicArgs;
UseListOrderMap UseListOrders;
SmallVector<StringRef, 8> MDNames;
/// Synchronization scope names registered with LLVMContext.
Expand Down Expand Up @@ -2946,7 +2952,8 @@ AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac,
ShouldPreserveUseListOrder(
PreserveAssemblyUseListOrder.getNumOccurrences()
? PreserveAssemblyUseListOrder
: ShouldPreserveUseListOrder) {
: ShouldPreserveUseListOrder),
PrettyPrintIntrinsicArgs(PrintFormattedIntrinsics) {
if (!TheModule)
return;
for (const GlobalObject &GO : TheModule->global_objects())
Expand All @@ -2958,7 +2965,8 @@ AssemblyWriter::AssemblyWriter(formatted_raw_ostream &o, SlotTracker &Mac,
const ModuleSummaryIndex *Index, bool IsForDebug)
: Out(o), TheIndex(Index), Machine(Mac), TypePrinter(/*Module=*/nullptr),
IsForDebug(IsForDebug),
ShouldPreserveUseListOrder(PreserveAssemblyUseListOrder) {}
ShouldPreserveUseListOrder(PreserveAssemblyUseListOrder),
PrettyPrintIntrinsicArgs(false) {}

void AssemblyWriter::writeOperand(const Value *Operand, bool PrintType) {
if (!Operand) {
Expand Down Expand Up @@ -4575,12 +4583,38 @@ void AssemblyWriter::printInstruction(const Instruction &I) {
Out << ' ';
writeOperand(Operand, false);
Out << '(';
bool HasPrettyPrintedArgs =
PrettyPrintIntrinsicArgs && isa<IntrinsicInst>(CI) &&
Intrinsic::hasPrettyPrintedArgs(CI->getIntrinsicID());

ListSeparator LS;
for (unsigned op = 0, Eop = CI->arg_size(); op < Eop; ++op) {
Out << LS;
writeParamOperand(CI->getArgOperand(op), PAL.getParamAttrs(op));
}
if (HasPrettyPrintedArgs) {
Function *CalledFunc = CI->getCalledFunction();
auto PrintArgComment = [&](unsigned ArgNo) {
const Constant *ConstArg = dyn_cast<Constant>(CI->getArgOperand(ArgNo));
if (!ConstArg)
return;
std::string ArgComment;
raw_string_ostream ArgCommentStream(ArgComment);
Intrinsic::ID IID = CalledFunc->getIntrinsicID();
Intrinsic::printImmArg(IID, ArgNo, ArgCommentStream, ConstArg);
if (ArgComment.empty())
return;
Out << "/* " << ArgComment << " */ ";
};

for (unsigned ArgNo = 0, NumArgs = CI->arg_size(); ArgNo < NumArgs;
++ArgNo) {
Out << LS;
PrintArgComment(ArgNo);
writeParamOperand(CI->getArgOperand(ArgNo), PAL.getParamAttrs(ArgNo));
}
} else {
for (unsigned op = 0, Eop = CI->arg_size(); op < Eop; ++op) {
Out << LS;
writeParamOperand(CI->getArgOperand(op), PAL.getParamAttrs(op));
}
}
// Emit an ellipsis if this is a musttail call in a vararg function. This
// is only to aid readability, musttail calls forward varargs by default.
if (CI->isMustTailCall() && CI->getParent() &&
Expand Down Expand Up @@ -5004,12 +5038,10 @@ void AssemblyWriter::printUseLists(const Function *F) {
//===----------------------------------------------------------------------===//

void Function::print(raw_ostream &ROS, AssemblyAnnotationWriter *AAW,
bool ShouldPreserveUseListOrder,
bool IsForDebug) const {
bool ShouldPreserveUseListOrder, bool IsForDebug) const {
SlotTracker SlotTable(this->getParent());
formatted_raw_ostream OS(ROS);
AssemblyWriter W(OS, SlotTable, this->getParent(), AAW,
IsForDebug,
AssemblyWriter W(OS, SlotTable, this->getParent(), AAW, IsForDebug,
ShouldPreserveUseListOrder);
W.printFunction(this);
}
Expand Down
11 changes: 11 additions & 0 deletions llvm/lib/IR/Intrinsics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include "llvm/IR/IntrinsicsX86.h"
#include "llvm/IR/IntrinsicsXCore.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/NVVMIntrinsicUtils.h"
#include "llvm/IR/Type.h"

using namespace llvm;
Expand Down Expand Up @@ -601,6 +602,12 @@ bool Intrinsic::isOverloaded(ID id) {
#undef GET_INTRINSIC_OVERLOAD_TABLE
}

bool Intrinsic::hasPrettyPrintedArgs(ID id){
#define GET_INTRINSIC_PRETTY_PRINT_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
#undef GET_INTRINSIC_PRETTY_PRINT_TABLE
}

/// Table of per-target intrinsic name tables.
#define GET_INTRINSIC_TARGET_DATA
#include "llvm/IR/IntrinsicImpl.inc"
Expand Down Expand Up @@ -1129,3 +1136,7 @@ Intrinsic::ID Intrinsic::getDeinterleaveIntrinsicID(unsigned Factor) {
assert(Factor >= 2 && Factor <= 8 && "Unexpected factor");
return InterleaveIntrinsics[Factor - 2].Deinterleave;
}

#define GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
#include "llvm/IR/IntrinsicImpl.inc"
#undef GET_INTRINSIC_PRETTY_PRINT_ARGUMENTS
41 changes: 41 additions & 0 deletions llvm/test/CodeGen/NVPTX/tcgen05-mma-tensor-formatted.ll
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; NOTE: This sample test demonstrates the --print-formatted-intrinsics feature for NVPTX intrinsics
; RUN: llvm-as < %s | llvm-dis --print-formatted-intrinsics | FileCheck %s

target triple = "nvptx64-nvidia-cuda"

define void @tcgen05_mma_fp16_cta1(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d) {
; CHECK-LABEL: define void @tcgen05_mma_fp16_cta1(
; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector=discard */ i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 0, i32 1, i32 0)

; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector=lastuse */ i32 1)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 0, i32 1, i32 1)

; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector=fill */ i32 2)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 0, i32 1, i32 2)

; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f16 */ i32 0, /* cta_group= */ i32 1, /* collector=use */ i32 3)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 0, i32 1, i32 3)

ret void
}

define void @tcgen05_mma_f8f6f4_cta2(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d) {
; CHECK-LABEL: define void @tcgen05_mma_f8f6f4_cta2(
; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector=discard */ i32 0)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 2, i32 2, i32 0)

; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector=lastuse */ i32 1)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 2, i32 2, i32 1)

; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector=fill */ i32 2)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 2, i32 2, i32 2)

; CHECK: call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, /* kind=f8f6f4 */ i32 2, /* cta_group= */ i32 2, /* collector=use */ i32 3)
call void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6) %dtmem, ptr addrspace(6) %atensor, i64 %b, i32 %idesc, i1 %enable_inp_d, i32 2, i32 2, i32 3)

ret void
}

declare void @llvm.nvvm.tcgen05.mma.tensor(ptr addrspace(6), ptr addrspace(6), i64, i32, i1, i32, i32, i32)
23 changes: 23 additions & 0 deletions llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,14 @@ void CodeGenIntrinsic::setProperty(const Record *R) {
int64_t Lower = R->getValueAsInt("Lower");
int64_t Upper = R->getValueAsInt("Upper");
addArgAttribute(ArgNo, Range, Lower, Upper);
} else if (R->isSubClassOf("ArgInfo")) {
unsigned ArgNo = R->getValueAsInt("ArgNo");
if (ArgNo < 1)
PrintFatalError(R->getLoc(),
"ArgInfo requires ArgNo >= 1 (0 is return value)");
StringRef ArgName = R->getValueAsString("ArgName");
StringRef FuncName = R->getValueAsString("FunctionName");
addPrettyPrintFunction(ArgNo - 1, ArgName, FuncName);
} else {
llvm_unreachable("Unknown property!");
}
Expand Down Expand Up @@ -468,3 +476,18 @@ void CodeGenIntrinsic::addArgAttribute(unsigned Idx, ArgAttrKind AK, uint64_t V,
ArgumentAttributes.resize(Idx + 1);
ArgumentAttributes[Idx].emplace_back(AK, V, V2);
}

void CodeGenIntrinsic::addPrettyPrintFunction(unsigned ArgIdx,
StringRef ArgName,
StringRef FuncName) {
auto It = llvm::find_if(PrettyPrintFunctions, [ArgIdx](const auto &Info) {
return Info.ArgIdx == ArgIdx;
});
if (It != PrettyPrintFunctions.end()) {
PrintFatalError(TheDef->getLoc(), "ArgInfo for argument " + Twine(ArgIdx) +
" is already defined as '" +
It->FuncName + "'");
return;
}
PrettyPrintFunctions.emplace_back(ArgIdx, ArgName, FuncName);
}
16 changes: 16 additions & 0 deletions llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,22 @@ struct CodeGenIntrinsic {
void addArgAttribute(unsigned Idx, ArgAttrKind AK, uint64_t V = 0,
uint64_t V2 = 0);

/// Structure to store pretty print and argument information.
struct PrettyPrintArgInfo {
unsigned ArgIdx;
StringRef ArgName;
StringRef FuncName;

PrettyPrintArgInfo(unsigned Idx, StringRef Name, StringRef Func)
: ArgIdx(Idx), ArgName(Name), FuncName(Func) {}
};

/// Vector that stores ArgInfo (ArgIndex, ArgName, FunctionName).
SmallVector<PrettyPrintArgInfo> PrettyPrintFunctions;

void addPrettyPrintFunction(unsigned ArgIdx, StringRef ArgName,
StringRef FuncName);

bool hasProperty(enum SDNP Prop) const { return Properties & (1 << Prop); }

/// Goes through all IntrProperties that have IsDefault value set and sets
Expand Down
Loading